diff --git a/java/google/registry/model/EntityClasses.java b/java/google/registry/model/EntityClasses.java index 0566a9771..6100d2866 100644 --- a/java/google/registry/model/EntityClasses.java +++ b/java/google/registry/model/EntityClasses.java @@ -23,6 +23,7 @@ import google.registry.model.billing.BillingEvent; import google.registry.model.billing.RegistrarBillingEntry; import google.registry.model.billing.RegistrarCredit; import google.registry.model.billing.RegistrarCreditBalance; +import google.registry.model.common.Cursor; import google.registry.model.common.EntityGroupRoot; import google.registry.model.common.GaeUserIdConverter; import google.registry.model.contact.ContactResource; @@ -77,6 +78,7 @@ public final class EntityClasses { CommitLogManifest.class, CommitLogMutation.class, ContactResource.class, + Cursor.class, DomainApplication.class, DomainApplicationIndex.class, DomainBase.class, diff --git a/java/google/registry/model/common/Cursor.java b/java/google/registry/model/common/Cursor.java new file mode 100644 index 000000000..a57ec8919 --- /dev/null +++ b/java/google/registry/model/common/Cursor.java @@ -0,0 +1,166 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model.common; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static google.registry.model.common.EntityGroupRoot.getCrossTldKey; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.util.DateTimeUtils.START_OF_TIME; + +import com.google.common.annotations.VisibleForTesting; + +import com.googlecode.objectify.Key; +import com.googlecode.objectify.annotation.Entity; +import com.googlecode.objectify.annotation.Id; +import com.googlecode.objectify.annotation.Parent; + +import google.registry.model.ImmutableObject; +import google.registry.model.registry.Registry; + +import org.joda.time.DateTime; + +/** + * Shared entity for date cursors. This type supports both "scoped" cursors (i.e. per resource + * of a given type, such as a TLD) and global (i.e. one per environment) cursors, defined internally + * as scoped on {@link EntityGroupRoot}. + */ +@Entity +public class Cursor extends ImmutableObject { + + /** The types of cursors, used as the string id field for each cursor in datastore. */ + public enum CursorType { + /** Cursor for ensuring rolling transactional isolation of BRDA staging operation. */ + BRDA(Registry.class), + + /** Cursor for ensuring rolling transactional isolation of RDE report operation. */ + RDE_REPORT(Registry.class), + + /** Cursor for ensuring rolling transactional isolation of RDE staging operation. */ + RDE_STAGING(Registry.class), + + /** Cursor for ensuring rolling transactional isolation of RDE upload operation. */ + RDE_UPLOAD(Registry.class), + + /** + * Cursor that tracks the last time we talked to the escrow provider's SFTP server for a given + * TLD. + * + *

Our escrow provider has an odd feature where separate deposits uploaded within two hours + * of each other will be merged into a single deposit. This is problematic in situations where + * the cursor might be a few days behind and is trying to catch up. + * + *

The way we solve this problem is by having {@code RdeUploadAction} check this cursor + * before performing an upload for a given TLD. If the cursor is less than two hours old, the + * action will fail with a status code above 300 and App Engine will keep retrying the action + * until it's ready. + */ + RDE_UPLOAD_SFTP(Registry.class), + + /** Cursor for ensuring rolling transactional isolation of recurring billing expansion. */ + RECURRING_BILLING(EntityGroupRoot.class); + + /** See the definition of scope on {@link #getScopeClass}. */ + private final Class scope; + + private CursorType(Class scope) { + this.scope = scope; + } + + /** + * If there are multiple cursors for a given cursor type, a cursor must also have a scope + * defined (distinct from a parent, which is always the EntityGroupRoot key). For instance, + * for a cursor that is defined at the registry level, the scope type will be Registry.class. + * For a cursor (theoretically) defined for each EPP resource, the scope type will be + * EppResource.class. For a global cursor, i.e. one that applies per environment, this will be + * {@link EntityGroupRoot}. + */ + public Class getScopeClass() { + return scope; + } + } + + @Parent + Key parent = getCrossTldKey(); + + @Id + String id; + + DateTime cursorTime = START_OF_TIME; + + /** + * Checks that the type of the scoped object (or null) matches the required type for the specified + * cursor (or null, if the cursor is a global cursor). + */ + private static void checkValidCursorTypeForScope( + CursorType cursorType, Key scope) { + checkArgument( + cursorType.getScopeClass().equals( + scope.equals(EntityGroupRoot.getCrossTldKey()) + ? EntityGroupRoot.class + : ofy().factory().getMetadata(scope).getEntityClass()), + "Class required for cursor does not match scope class"); + } + + /** Generates a unique ID for a given scope key and cursor type. */ + private static String generateId(CursorType cursorType, Key scope) { + return String.format("%s_%s", scope.getString(), cursorType.name()); + } + + /** Creates a unique key for a given scope and cursor type. */ + @VisibleForTesting + static Key createKey(CursorType cursorType, ImmutableObject scope) { + Key scopeKey = Key.create(scope); + checkValidCursorTypeForScope(cursorType, scopeKey); + return Key.create(getCrossTldKey(), Cursor.class, generateId(cursorType, scopeKey)); + } + + /** Creates a unique key for a given global cursor type. */ + @VisibleForTesting + static Key createGlobalKey(CursorType cursorType) { + checkArgument( + cursorType.getScopeClass().equals(EntityGroupRoot.class), + "Cursor type is not a global cursor."); + return Key.create( + getCrossTldKey(), Cursor.class, generateId(cursorType, EntityGroupRoot.getCrossTldKey())); + } + + /** Creates a new global cursor instance. */ + public static Cursor createGlobal(CursorType cursorType, DateTime cursorTime) { + return create(cursorType, cursorTime, EntityGroupRoot.getCrossTldKey()); + } + + /** Creates a new cursor instance with a given {@link Key} scope. */ + private static Cursor create( + CursorType cursorType, DateTime cursorTime, Key scope) { + Cursor instance = new Cursor(); + instance.cursorTime = checkNotNull(cursorTime, "Cursor time cannot be null"); + checkNotNull(scope, "Cursor scope cannot be null"); + checkNotNull(cursorType, "Cursor type cannot be null"); + checkValidCursorTypeForScope(cursorType, scope); + instance.id = generateId(cursorType, scope); + return instance; + } + + /** Creates a new cursor instance with a given {@link ImmutableObject} scope. */ + public static Cursor create(CursorType cursorType, DateTime cursorTime, ImmutableObject scope) { + checkNotNull(scope, "Cursor scope cannot be null"); + return create(cursorType, cursorTime, Key.create(scope)); + } + + public DateTime getCursorTime() { + return cursorTime; + } +} diff --git a/java/google/registry/model/registry/RegistryCursor.java b/java/google/registry/model/registry/RegistryCursor.java index 51006181a..d3d9db829 100644 --- a/java/google/registry/model/registry/RegistryCursor.java +++ b/java/google/registry/model/registry/RegistryCursor.java @@ -24,13 +24,17 @@ import com.googlecode.objectify.annotation.Id; import com.googlecode.objectify.annotation.Parent; import google.registry.model.ImmutableObject; +import google.registry.model.common.Cursor; import org.joda.time.DateTime; -/** Shared entity type for per-TLD date cursors. */ +/** Shared entity for per-TLD date cursors. */ @Entity public class RegistryCursor extends ImmutableObject { + // TODO(b/28386088): Drop this class once all registry cursors have been saved in parallel as + // new-style Cursors (either through business-as-usual operations or UpdateCursorsCommand). + /** The types of cursors, used as the string id field for each cursor in datastore. */ public enum CursorType { /** Cursor for ensuring rolling transactional isolation of BRDA staging operation. */ @@ -80,6 +84,10 @@ public class RegistryCursor extends ImmutableObject { /** Convenience shortcut to save a cursor. */ public static void save(Registry registry, CursorType cursorType, DateTime value) { ofy().save().entity(create(registry, cursorType, value)); + // In parallel, save the new cursor type alongside the old. + ofy() + .save() + .entity(Cursor.create(Cursor.CursorType.valueOf(cursorType.name()), value, registry)); } /** Creates a new cursor instance. */ diff --git a/java/google/registry/tools/UpdateCursorsCommand.java b/java/google/registry/tools/UpdateCursorsCommand.java index 105c3296b..434475a5c 100644 --- a/java/google/registry/tools/UpdateCursorsCommand.java +++ b/java/google/registry/tools/UpdateCursorsCommand.java @@ -19,6 +19,7 @@ import com.google.common.base.Optional; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import google.registry.model.common.Cursor; import google.registry.model.registry.Registry; import google.registry.model.registry.RegistryCursor; import google.registry.model.registry.RegistryCursor.CursorType; @@ -32,6 +33,8 @@ import java.util.List; @Parameters(separators = " =", commandDescription = "Modifies cursor timestamps used by LRC tasks") final class UpdateCursorsCommand extends MutatingCommand { + // TODO(b/28386088): Cut command over to new Cursor format + @Parameter( description = "TLDs on which to operate.", required = true) @@ -60,6 +63,10 @@ final class UpdateCursorsCommand extends MutatingCommand { ? RegistryCursor.create(registry, cursorType, expectedTimestamp.get()) : null, RegistryCursor.create(registry, cursorType, newTimestamp)); - } + Cursor.CursorType newCursorType = Cursor.CursorType.valueOf(cursorType.name()); + stageEntityChange( + null, + Cursor.create(newCursorType, newTimestamp, registry)); + } } } diff --git a/javatests/google/registry/export/backup_kinds.txt b/javatests/google/registry/export/backup_kinds.txt index dc4dcc59a..bd5f652bc 100644 --- a/javatests/google/registry/export/backup_kinds.txt +++ b/javatests/google/registry/export/backup_kinds.txt @@ -1,6 +1,7 @@ Cancellation ContactResource +Cursor DomainApplicationIndex DomainBase EntityGroupRoot diff --git a/javatests/google/registry/model/common/CursorTest.java b/javatests/google/registry/model/common/CursorTest.java new file mode 100644 index 000000000..86dd79371 --- /dev/null +++ b/javatests/google/registry/model/common/CursorTest.java @@ -0,0 +1,154 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model.common; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.model.common.Cursor.CursorType.BRDA; +import static google.registry.model.common.Cursor.CursorType.RDE_UPLOAD; +import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.testing.DatastoreHelper.createTld; +import static google.registry.testing.DatastoreHelper.persistActiveDomain; +import static google.registry.util.DateTimeUtils.START_OF_TIME; + +import com.googlecode.objectify.VoidWork; + +import google.registry.model.EntityTestCase; +import google.registry.model.domain.DomainResource; +import google.registry.model.registry.Registry; +import google.registry.testing.ExceptionRule; + +import org.joda.time.DateTime; +import org.junit.Rule; +import org.junit.Test; + +/** Unit tests for {@link Cursor}. */ +public class CursorTest extends EntityTestCase { + + @Rule public final ExceptionRule thrown = new ExceptionRule(); + + @Test + public void testSuccess_persistScopedCursor() { + createTld("tld"); + clock.advanceOneMilli(); + final DateTime time = DateTime.parse("2012-07-12T03:30:00.000Z"); + ofy() + .transact( + new VoidWork() { + @Override + public void vrun() { + ofy().save().entity(Cursor.create(RDE_UPLOAD, time, Registry.get("tld"))); + } + }); + assertThat(ofy().load().key(Cursor.createKey(BRDA, Registry.get("tld"))).now()).isNull(); + assertThat( + ofy() + .load() + .key(Cursor.createKey(RDE_UPLOAD, Registry.get("tld"))) + .now() + .getCursorTime()) + .isEqualTo(time); + } + + @Test + public void testSuccess_persistGlobalCursor() { + final DateTime time = DateTime.parse("2012-07-12T03:30:00.000Z"); + ofy() + .transact( + new VoidWork() { + @Override + public void vrun() { + ofy().save().entity(Cursor.createGlobal(RECURRING_BILLING, time)); + } + }); + assertThat(ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now().getCursorTime()) + .isEqualTo(time); + } + + @Test + public void testIndexing() throws Exception { + final DateTime time = DateTime.parse("2012-07-12T03:30:00.000Z"); + ofy() + .transact( + new VoidWork() { + @Override + public void vrun() { + ofy().save().entity(Cursor.createGlobal(RECURRING_BILLING, time)); + } + }); + Cursor cursor = ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now(); + verifyIndexing(cursor); + } + + @Test + public void testFailure_invalidScopeOnCreate() throws Exception { + createTld("tld"); + clock.advanceOneMilli(); + final DateTime time = DateTime.parse("2012-07-12T03:30:00.000Z"); + final DomainResource domain = persistActiveDomain("notaregistry.tld"); + thrown.expect( + IllegalArgumentException.class, "Class required for cursor does not match scope class"); + ofy() + .transact( + new VoidWork() { + @Override + public void vrun() { + ofy().save().entity(Cursor.create(RDE_UPLOAD, time, domain)); + } + }); + } + + @Test + public void testFailure_invalidScopeOnKeyCreate() throws Exception { + createTld("tld"); + thrown.expect( + IllegalArgumentException.class, "Class required for cursor does not match scope class"); + Cursor.createKey(RDE_UPLOAD, persistActiveDomain("notaregistry.tld")); + } + + @Test + public void testFailure_createGlobalKeyForScopedCursorType() throws Exception { + thrown.expect(IllegalArgumentException.class, "Cursor type is not a global cursor"); + Cursor.createGlobalKey(RDE_UPLOAD); + } + + @Test + public void testFailure_invalidScopeOnGlobalKeyCreate() throws Exception { + createTld("tld"); + thrown.expect( + IllegalArgumentException.class, "Class required for cursor does not match scope class"); + Cursor.createKey(RECURRING_BILLING, persistActiveDomain("notaregistry.tld")); + } + + @Test + public void testFailure_nullScope() throws Exception { + thrown.expect(NullPointerException.class, "Cursor scope cannot be null"); + Cursor.create(RECURRING_BILLING, START_OF_TIME, null); + } + + @Test + public void testFailure_nullCursorType() throws Exception { + createTld("tld"); + thrown.expect(NullPointerException.class, "Cursor type cannot be null"); + Cursor.create(null, START_OF_TIME, Registry.get("tld")); + } + + @Test + public void testFailure_nullTime() throws Exception { + createTld("tld"); + thrown.expect(NullPointerException.class, "Cursor time cannot be null"); + Cursor.create(RDE_UPLOAD, null, Registry.get("tld")); + } +} diff --git a/javatests/google/registry/rde/testdata/deposit_full.xml b/javatests/google/registry/rde/testdata/deposit_full.xml index 4d9029d3b..70f607571 100644 --- a/javatests/google/registry/rde/testdata/deposit_full.xml +++ b/javatests/google/registry/rde/testdata/deposit_full.xml @@ -56,7 +56,7 @@ - + example1.test Dexample1-TEST @@ -74,7 +74,7 @@ 2015-04-03T22:00:00.0Z - + example2.test Dexample2-TEST