mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-09-12 11:06:29 +00:00
Code Issues Releases Wiki Activity

Remove Ofy (#1863)

Browse Source 
So long, farewell, adios, ciao, sayonara, 再见!

TESTED=deployed to alpha and used `nomulus list_tlds` to confirm that the web app can receive and serve requests.

<!-- Reviewable:start -->
- - -
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/google/nomulus/1863)
<!-- Reviewable:end -->
This commit is contained in:
Lai Jiang
2022-12-02 22:28:33 -05:00
committed by GitHub
parent 601aed388c
commit 1d7dfe4e07
58 changed files with 31 additions and 2381 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java
-5
View File
@@ -21,7 +21,6 @@ import com.google.common.flogger.FluentLogger;
import dagger.Lazy;
import google.registry.config.RegistryEnvironment;
import google.registry.config.SystemPropertySetter;
import google.registry.model.AppEngineEnvironment;
import google.registry.model.IdService;
import google.registry.persistence.transaction.JpaTransactionManager;
import google.registry.persistence.transaction.TransactionManagerFactory;
@@ -63,10 +62,6 @@ public class RegistryPipelineWorkerInitializer implements JvmInitializer {
transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager();
}
TransactionManagerFactory.setJpaTmOnBeamWorker(transactionManagerLazy::get);
// Masquerade all threads as App Engine threads, so we can create Ofy keys in the pipeline. Also
// loads all ofy entities.
new AppEngineEnvironment("s~" + registryPipelineComponent.getProjectId())
.setEnvironmentForAllThreads();
SystemPropertySetter.PRODUCTION_IMPL.setProperty(PROPERTY, "true");
// Use self-allocated IDs if requested. Note that this inevitably results in duplicate IDs from
// multiple workers, which can also collide with existing IDs in the database. So they cannot be
core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml
Vendored
-20
View File
@@ -341,24 +341,4 @@ have been in the database for a certain period of time. -->
<transport-guarantee>CONFIDENTIAL</transport-guarantee>
</user-data-constraint>
</security-constraint>
<!-- See: https://code.google.com/p/objectify-appengine/wiki/Setup -->
<filter>
<filter-name>ObjectifyFilter</filter-name>
<filter-class>com.googlecode.objectify.ObjectifyFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>ObjectifyFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
<!-- Register types with Objectify. -->
<filter>
<filter-name>OfyFilter</filter-name>
<filter-class>google.registry.model.ofy.OfyFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>OfyFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
</web-app>
core/src/main/java/google/registry/env/common/default/WEB-INF/web.xml
Vendored
-20
View File
@@ -139,24 +139,4 @@
<transport-guarantee>CONFIDENTIAL</transport-guarantee>
</user-data-constraint>
</security-constraint>
<!-- See: https://code.google.com/p/objectify-appengine/wiki/Setup -->
<filter>
<filter-name>ObjectifyFilter</filter-name>
<filter-class>com.googlecode.objectify.ObjectifyFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>ObjectifyFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
<!-- Register types with Objectify. -->
<filter>
<filter-name>OfyFilter</filter-name>
<filter-class>google.registry.model.ofy.OfyFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>OfyFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
</web-app>
core/src/main/java/google/registry/env/common/pubapi/WEB-INF/web.xml
Vendored
-20
View File
@@ -105,24 +105,4 @@
<transport-guarantee>CONFIDENTIAL</transport-guarantee>
</user-data-constraint>
</security-constraint>
<!-- See: https://code.google.com/p/objectify-appengine/wiki/Setup -->
<filter>
<filter-name>ObjectifyFilter</filter-name>
<filter-class>com.googlecode.objectify.ObjectifyFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>ObjectifyFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
<!-- Register types with Objectify. -->
<filter>
<filter-name>OfyFilter</filter-name>
<filter-class>google.registry.model.ofy.OfyFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>OfyFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
</web-app>
core/src/main/java/google/registry/env/common/tools/WEB-INF/web.xml
Vendored
-20
View File
@@ -135,24 +135,4 @@
<transport-guarantee>CONFIDENTIAL</transport-guarantee>
</user-data-constraint>
</security-constraint>
<!-- See: https://code.google.com/p/objectify-appengine/wiki/Setup -->
<filter>
<filter-name>ObjectifyFilter</filter-name>
<filter-class>com.googlecode.objectify.ObjectifyFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>ObjectifyFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
<!-- Register types with Objectify. -->
<filter>
<filter-name>OfyFilter</filter-name>
<filter-class>google.registry.model.ofy.OfyFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>OfyFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
</web-app>
core/src/main/java/google/registry/model/AppEngineEnvironment.java
-124
View File
@@ -1,124 +0,0 @@
// Copyright 2020 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model;
import com.google.apphosting.api.ApiProxy;
import com.google.apphosting.api.ApiProxy.Environment;
import com.google.common.collect.ImmutableMap;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.model.ofy.ObjectifyService;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
/**
* Sets up a fake {@link Environment} so that the following operations can be performed without the
* Datastore service:
*
* <ul>
* <li>Create Objectify {@code Keys}.
* <li>Instantiate Objectify objects.
* <li>Convert Datastore {@code Entities} to their corresponding Objectify objects.
* </ul>
*
* <p>User has the option to specify their desired {@code appId} string, which forms part of an
* Objectify {@code Key} and is included in the equality check. This feature makes it easy to
* compare a migrated object in SQL with the original in Objectify.
*
* <p>Note that conversion from Objectify objects to Datastore {@code Entities} still requires the
* Datastore service.
*/
@DeleteAfterMigration
public class AppEngineEnvironment {
private Environment environment;
/**
* Constructor for use by tests.
*
* <p>All test suites must use the same appId for environments, since when tearing down we do not
* clear cached environments in spawned threads. See {@link #unsetEnvironmentForAllThreads} for
* more information.
*/
public AppEngineEnvironment() {
/**
* Use AppEngineExtension's appId here so that ofy and sql entities can be compared with {@code
* Objects#equals()}. The choice of this value does not impact functional correctness.
*/
this("test");
}
/** Constructor for use by applications, e.g., BEAM pipelines. */
public AppEngineEnvironment(String appId) {
environment = createAppEngineEnvironment(appId);
}
public void setEnvironmentForCurrentThread() {
ApiProxy.setEnvironmentForCurrentThread(environment);
ObjectifyService.initOfy();
}
public void setEnvironmentForAllThreads() {
setEnvironmentForCurrentThread();
ApiProxy.setEnvironmentFactory(() -> environment);
}
public void unsetEnvironmentForCurrentThread() {
ApiProxy.clearEnvironmentForCurrentThread();
}
/**
* Unsets the test environment in all threads with best effort.
*
* <p>This method unsets the environment factory and clears the cached environment in the current
* thread (the main test runner thread). We do not clear the cache in spawned threads, even though
* they may be reused. This is not a problem as long as the appId stays the same: those threads
* are used only in AppEngine or BEAM tests, and expect the presence of an environment.
*/
public void unsetEnvironmentForAllThreads() {
unsetEnvironmentForCurrentThread();
try {
Method method = ApiProxy.class.getDeclaredMethod("clearEnvironmentFactory");
method.setAccessible(true);
method.invoke(null);
} catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
throw new RuntimeException(e);
}
}
/** Returns a placeholder {@link Environment} that can return hardcoded AppId and Attributes. */
private static Environment createAppEngineEnvironment(String appId) {
return (Environment)
Proxy.newProxyInstance(
Environment.class.getClassLoader(),
new Class[] {Environment.class},
(Object proxy, Method method, Object[] args) -> {
switch (method.getName()) {
case "getAppId":
return appId;
case "getAttributes":
return ImmutableMap.<String, Object>of();
default:
throw new UnsupportedOperationException(method.getName());
}
});
}
/** Returns true if the current thread is in an App Engine Environment. */
public static boolean isInAppEngineEnvironment() {
return ApiProxy.getCurrentEnvironment() != null;
}
}
core/src/main/java/google/registry/model/CacheUtils.java
-29
View File
@@ -18,13 +18,9 @@ import static com.google.common.base.Suppliers.memoizeWithExpiration;
import static google.registry.config.RegistryConfig.getSingletonCacheRefreshDuration;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.base.Supplier;
import google.registry.model.annotations.DeleteAfterMigration;
import java.time.Duration;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
/** Utility methods related to caching Datastore entities. */
public class CacheUtils {
@@ -77,29 +73,4 @@ public class CacheUtils {
}
return caffeine;
}
/**
* A {@link CacheLoader} that automatically masquerade the background thread where the refresh
* action runs in to be an GAE thread.
*/
@DeleteAfterMigration
public abstract static class AppEngineEnvironmentCacheLoader<K, V> implements CacheLoader<K, V> {
private static final AppEngineEnvironment environment = new AppEngineEnvironment();
@Override
public @Nullable V reload(@NonNull K key, @NonNull V oldValue) throws Exception {
V value;
boolean isMasqueraded = false;
if (!AppEngineEnvironment.isInAppEngineEnvironment()) {
environment.setEnvironmentForCurrentThread();
isMasqueraded = true;
}
value = load(key);
if (isMasqueraded) {
environment.unsetEnvironmentForCurrentThread();
}
return value;
}
}
}
core/src/main/java/google/registry/model/EppResource.java
+1 -2
View File
@@ -32,7 +32,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import google.registry.config.RegistryConfig;
import google.registry.model.CacheUtils.AppEngineEnvironmentCacheLoader;
import google.registry.model.annotations.IdAllocation;
import google.registry.model.eppcommon.StatusValue;
import google.registry.model.transfer.TransferData;
@@ -354,7 +353,7 @@ public abstract class EppResource extends UpdateAutoTimestampEntity implements B
}
static final CacheLoader<VKey<? extends EppResource>, EppResource> CACHE_LOADER =
new AppEngineEnvironmentCacheLoader<VKey<? extends EppResource>, EppResource>() {
new CacheLoader<VKey<? extends EppResource>, EppResource>() {
@Override
public EppResource load(VKey<? extends EppResource> key) {
core/src/main/java/google/registry/model/IdService.java
-2
View File
@@ -22,7 +22,6 @@ import com.google.appengine.api.datastore.DatastoreServiceFactory;
import com.google.common.flogger.FluentLogger;
import google.registry.beam.common.RegistryPipelineWorkerInitializer;
import google.registry.config.RegistryEnvironment;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.model.common.DatabaseMigrationStateSchedule;
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
import java.math.BigInteger;
@@ -33,7 +32,6 @@ import org.joda.time.DateTime;
/**
* Allocates a {@link long} to use as a {@code @Id}, (part) of the primary SQL key for an entity.
*/
@DeleteAfterMigration
public final class IdService {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
core/src/main/java/google/registry/model/annotations/NotBackedUp.java
-47
View File
@@ -1,47 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.annotations;
import com.googlecode.objectify.annotation.Entity;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Annotation for an Objectify {@link Entity} to indicate that it should not be backed up by the
* default Datastore backup configuration (it may be backed up by something else).
*/
@DeleteAfterMigration
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
public @interface NotBackedUp {
Reason reason();
/** Reasons why a given entity does not need to be be backed up. */
enum Reason {
/** This entity is transient by design and has only a short-term useful lifetime. */
TRANSIENT,
/** This entity's data is already regularly pulled down from an external source. */
EXTERNALLY_SOURCED,
/** This entity is generated automatically by the app and will be recreated if need be. */
AUTO_GENERATED,
/** Commit log entities are exported separately from the regular backups, by design. */
COMMIT_LOGS
}
}
core/src/main/java/google/registry/model/annotations/VirtualEntity.java
-32
View File
@@ -1,32 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.annotations;
import com.googlecode.objectify.annotation.Entity;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* Annotation for an Objectify {@link Entity} to indicate that it is a "virtual entity".
*
* <p>A virtual entity type exists only to define part of the parentage key hierarchy for its child
* entities, and is never actually persisted and thus has no fields besides its ID field.
*/
@DeleteAfterMigration
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
public @interface VirtualEntity {}
core/src/main/java/google/registry/model/common/GaeUserIdConverter.java
-73
View File
@@ -1,73 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.common;
import static com.google.common.base.Preconditions.checkState;
import static google.registry.model.IdService.allocateId;
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import com.google.appengine.api.users.User;
import com.google.common.base.Splitter;
import com.googlecode.objectify.annotation.Entity;
import com.googlecode.objectify.annotation.Id;
import google.registry.model.ImmutableObject;
import google.registry.model.annotations.NotBackedUp;
import google.registry.model.annotations.NotBackedUp.Reason;
import java.util.List;
/**
* A helper class to convert email addresses to GAE user ids. It does so by persisting a User
* object with the email address to Datastore, and then immediately reading it back.
*/
@Entity
@NotBackedUp(reason = Reason.TRANSIENT)
public class GaeUserIdConverter extends ImmutableObject {
@Id
public long id;
User user;
/**
* Converts an email address to a GAE user id.
*
* @return Numeric GAE user id (in String form), or null if email address has no GAE id
*/
public static String convertEmailAddressToGaeUserId(String emailAddress) {
final GaeUserIdConverter gaeUserIdConverter = new GaeUserIdConverter();
gaeUserIdConverter.id = allocateId();
List<String> emailParts = Splitter.on('@').splitToList(emailAddress);
checkState(emailParts.size() == 2, "'%s' is not a valid email address", emailAddress);
gaeUserIdConverter.user = new User(emailAddress, emailParts.get(1));
try {
// Perform these operations in a transactionless context to avoid enlisting in some outer
// transaction (if any).
auditedOfy()
.doTransactionless(
() -> {
auditedOfy().saveWithoutBackup().entity(gaeUserIdConverter).now();
return null;
});
// The read must be done in its own transaction to avoid reading from the session cache.
return auditedOfy()
.transactNew(() -> auditedOfy().load().entity(gaeUserIdConverter).now().user.getUserId());
} finally {
auditedOfy()
.doTransactionless(
() -> auditedOfy().deleteWithoutBackup().entity(gaeUserIdConverter).now());
}
}
}
core/src/main/java/google/registry/model/ofy/AugmentedDeleter.java
-90
View File
@@ -1,90 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.ofy;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.googlecode.objectify.ObjectifyService.ofy;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Streams;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.Result;
import com.googlecode.objectify.cmd.DeleteType;
import com.googlecode.objectify.cmd.Deleter;
import google.registry.model.annotations.DeleteAfterMigration;
import java.util.Arrays;
import java.util.stream.Stream;
/**
* A Deleter that forwards to {@code auditedOfy().delete()}, but can be augmented via subclassing to
* do custom processing on the keys to be deleted prior to their deletion.
*/
@DeleteAfterMigration
abstract class AugmentedDeleter implements Deleter {
private final Deleter delegate = ofy().delete();
/** Extension method to allow this Deleter to do extra work prior to the actual delete. */
protected abstract void handleDeletion(Iterable<Key<?>> keys);
private void handleDeletionStream(Stream<?> entityStream) {
handleDeletion(entityStream.map(Key::create).collect(toImmutableList()));
}
@Override
public Result<Void> entities(Iterable<?> entities) {
handleDeletionStream(Streams.stream(entities));
return delegate.entities(entities);
}
@Override
public Result<Void> entities(Object... entities) {
handleDeletionStream(Arrays.stream(entities));
return delegate.entities(entities);
}
@Override
public Result<Void> entity(Object entity) {
handleDeletionStream(Stream.of(entity));
return delegate.entity(entity);
}
@Override
public Result<Void> key(Key<?> key) {
handleDeletion(ImmutableList.of(key));
return delegate.keys(key);
}
@Override
public Result<Void> keys(Iterable<? extends Key<?>> keys) {
// Magic to convert the type Iterable<? extends Key<?>> (a family of types which allows for
// homogeneous iterables of a fixed Key<T> type, e.g. List<Key<Lock>>, and is convenient for
// callers) into the type Iterable<Key<?>> (a concrete type of heterogeneous keys, which is
// convenient for users).
handleDeletion(ImmutableList.copyOf(keys));
return delegate.keys(keys);
}
@Override
public Result<Void> keys(Key<?>... keys) {
handleDeletion(Arrays.asList(keys));
return delegate.keys(keys);
}
/** Augmenting this gets ugly; you can always just use keys(Key.create(...)) instead. */
@Override
public DeleteType type(Class<?> clazz) {
throw new UnsupportedOperationException();
}
}
core/src/main/java/google/registry/model/ofy/AugmentedSaver.java
-63
View File
@@ -1,63 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.ofy;
import static com.googlecode.objectify.ObjectifyService.ofy;
import com.google.appengine.api.datastore.Entity;
import com.google.common.collect.ImmutableList;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.Result;
import com.googlecode.objectify.cmd.Saver;
import google.registry.model.annotations.DeleteAfterMigration;
import java.util.Arrays;
import java.util.Map;
/**
* A Saver that forwards to {@code ofy().save()}, but can be augmented via subclassing to do custom
* processing on the entities to be saved prior to their saving.
*/
@DeleteAfterMigration
abstract class AugmentedSaver implements Saver {
private final Saver delegate = ofy().save();
/** Extension method to allow this Saver to do extra work prior to the actual save. */
protected abstract void handleSave(Iterable<?> entities);
@Override
public <E> Result<Map<Key<E>, E>> entities(Iterable<E> entities) {
handleSave(entities);
return delegate.entities(entities);
}
@Override
@SafeVarargs
public final <E> Result<Map<Key<E>, E>> entities(E... entities) {
handleSave(Arrays.asList(entities));
return delegate.entities(entities);
}
@Override
public <E> Result<Key<E>> entity(E entity) {
handleSave(ImmutableList.of(entity));
return delegate.entity(entity);
}
@Override
public Entity toEntity(Object pojo) {
// No call to the extension method, since toEntity() doesn't do any actual saving.
return delegate.toEntity(pojo);
}
}
core/src/main/java/google/registry/model/ofy/CommitLoggedWork.java
-84
View File
@@ -1,84 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.ofy;
import static com.google.common.base.Preconditions.checkState;
import com.google.common.collect.ImmutableSet;
import google.registry.model.ImmutableObject;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.util.Clock;
import java.util.function.Supplier;
/** Wrapper for {@link Supplier} that associates a time with each attempt. */
@DeleteAfterMigration
public class CommitLoggedWork<R> implements Runnable {
private final Supplier<R> work;
private final Clock clock;
/**
* Temporary place to store the result of a non-void work.
*
* <p>We don't want to return the result directly because we are going to try to recover from a
* {@link com.google.appengine.api.datastore.DatastoreTimeoutException} deep inside Objectify when
* it tries to commit the transaction. When an exception is thrown the return value would be lost,
* but sometimes we will be able to determine that we actually succeeded despite the timeout, and
* we'll want to get the result.
*/
private R result;
/**
* Temporary place to store the mutations belonging to the commit log manifest.
*
* <p>These are used along with the manifest to determine whether a transaction succeeded.
*/
protected ImmutableSet<ImmutableObject> mutations = ImmutableSet.of();
/** Lifecycle marker to track whether {@link #run} has been called. */
private boolean runCalled;
CommitLoggedWork(Supplier<R> work, Clock clock) {
this.work = work;
this.clock = clock;
}
protected TransactionInfo createNewTransactionInfo() {
return new TransactionInfo(clock.nowUtc());
}
boolean hasRun() {
return runCalled;
}
R getResult() {
checkState(runCalled, "Cannot call getResult() before run()");
return result;
}
@Override
public void run() {
// The previous time will generally be null, except when using transactNew.
TransactionInfo previous = Ofy.TRANSACTION_INFO.get();
// Set the time to be used for "now" within the transaction.
try {
Ofy.TRANSACTION_INFO.set(createNewTransactionInfo());
result = work.get();
} finally {
Ofy.TRANSACTION_INFO.set(previous);
}
runCalled = true;
}
}
core/src/main/java/google/registry/model/ofy/ObjectifyService.java
-140
View File
@@ -1,140 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.ofy;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static com.googlecode.objectify.ObjectifyService.factory;
import static google.registry.util.TypeUtils.hasAnnotation;
import com.google.appengine.api.datastore.AsyncDatastoreService;
import com.google.appengine.api.datastore.DatastoreServiceConfig;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Streams;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.ObjectifyFactory;
import com.googlecode.objectify.annotation.Entity;
import com.googlecode.objectify.annotation.EntitySubclass;
import google.registry.config.RegistryEnvironment;
import google.registry.model.Buildable;
import google.registry.model.ImmutableObject;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.model.common.GaeUserIdConverter;
/**
* An instance of Ofy, obtained via {@code #auditedOfy()}, should be used to access all persistable
* objects. The class contains a static initializer to call factory().register(...) on all
* persistable objects in this package.
*/
@DeleteAfterMigration
public class ObjectifyService {
/** A singleton instance of our Ofy wrapper. */
private static final Ofy OFY = new Ofy(null);
/**
* Returns the singleton {@link Ofy} instance, signifying that the caller has been audited for the
* Registry 3.0 conversion.
*/
public static Ofy auditedOfy() {
return OFY;
}
static {
initOfyOnce();
}
/** Ensures that Objectify has been fully initialized. */
public static void initOfy() {
// This method doesn't actually do anything; it's here so that callers have something to call
// to ensure that the static initialization of ObjectifyService has been performed (which Java
// guarantees will happen exactly once, before any static methods are invoked).
//
// See JLS section 12.4: http://docs.oracle.com/javase/specs/jls/se7/html/jls-12.html#jls-12.4
}
/**
* Performs static initialization for Objectify to register types and do other setup.
*
* <p>This method is non-idempotent, so it should only be called exactly once, which is achieved
* by calling it from this class's static initializer block.
*/
private static void initOfyOnce() {
// Set an ObjectifyFactory that uses our extended ObjectifyImpl.
// The "false" argument means that we are not using the v5-style Objectify embedded entities.
com.googlecode.objectify.ObjectifyService.setFactory(
new ObjectifyFactory(false) {
@Override
protected AsyncDatastoreService createRawAsyncDatastoreService(
DatastoreServiceConfig cfg) {
// In the unit test environment, wrap the Datastore service in a proxy that can be used
// to examine the number of requests sent to Datastore.
AsyncDatastoreService service = super.createRawAsyncDatastoreService(cfg);
return RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)
? new RequestCapturingAsyncDatastoreService(service)
: service;
}
});
registerEntityClasses(ImmutableSet.of(GaeUserIdConverter.class));
}
/** Register classes that can be persisted via Objectify as Datastore entities. */
private static void registerEntityClasses(
ImmutableSet<Class<? extends ImmutableObject>> entityClasses) {
// Register all the @Entity classes before any @EntitySubclass classes so that we can check
// that every @Entity registration is a new kind and every @EntitySubclass registration is not.
// This is future-proofing for Objectify 5.x where the registration logic gets less lenient.
for (Class<?> clazz :
Streams.concat(
entityClasses.stream().filter(hasAnnotation(Entity.class)),
entityClasses.stream().filter(hasAnnotation(Entity.class).negate()))
.collect(toImmutableSet())) {
String kind = Key.getKind(clazz);
boolean registered = factory().getMetadata(kind) != null;
if (clazz.isAnnotationPresent(Entity.class)) {
// Objectify silently replaces current registration for a given kind string when a different
// class is registered again for this kind. For simplicity's sake, throw an exception on any
// re-registration.
checkState(
!registered,
"Kind '%s' already registered, cannot register new @Entity %s",
kind,
clazz.getCanonicalName());
} else if (clazz.isAnnotationPresent(EntitySubclass.class)) {
// Ensure that any @EntitySubclass classes have also had their parent @Entity registered,
// which Objectify nominally requires but doesn't enforce in 4.x (though it may in 5.x).
checkState(
registered,
"No base entity for kind '%s' registered yet, cannot register new @EntitySubclass %s",
kind,
clazz.getCanonicalName());
}
com.googlecode.objectify.ObjectifyService.register(clazz);
// Autogenerated ids make the commit log code very difficult since we won't always be able
// to create a key for an entity immediately when requesting a save. So, we require such
// entities to implement google.registry.model.Buildable as its build() function allocates the
// id to the entity.
if (factory().getMetadata(clazz).getKeyMetadata().isIdGeneratable()) {
checkState(
Buildable.class.isAssignableFrom(clazz),
"Can't register %s: Entity with autogenerated ids (@Id on a Long) must implement"
+ " google.registry.model.Buildable.",
kind);
}
}
}
}
core/src/main/java/google/registry/model/ofy/Ofy.java
-374
View File
@@ -1,374 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.ofy;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Maps.uniqueIndex;
import static com.googlecode.objectify.ObjectifyService.ofy;
import static google.registry.config.RegistryConfig.getBaseOfyRetryDuration;
import com.google.appengine.api.datastore.DatastoreFailureException;
import com.google.appengine.api.datastore.DatastoreTimeoutException;
import com.google.appengine.api.taskqueue.TransientFailureException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Streams;
import com.google.common.flogger.FluentLogger;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.Objectify;
import com.googlecode.objectify.ObjectifyFactory;
import com.googlecode.objectify.cmd.Deleter;
import com.googlecode.objectify.cmd.Loader;
import com.googlecode.objectify.cmd.Saver;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.model.annotations.NotBackedUp;
import google.registry.model.annotations.VirtualEntity;
import google.registry.model.ofy.ReadOnlyWork.KillTransactionException;
import google.registry.util.Clock;
import google.registry.util.NonFinalForTesting;
import google.registry.util.Sleeper;
import google.registry.util.SystemClock;
import google.registry.util.SystemSleeper;
import java.lang.annotation.Annotation;
import java.util.Objects;
import java.util.function.Supplier;
import javax.inject.Inject;
import org.joda.time.DateTime;
import org.joda.time.Duration;
/**
* A wrapper around ofy().
*
* <p>The primary purpose of this class is to add functionality to support commit logs. It is
* simpler to wrap {@link Objectify} rather than extend it because this way we can remove some
* methods that we don't really want exposed and add some shortcuts.
*/
@DeleteAfterMigration
public class Ofy {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
/** Default clock for transactions that don't provide one. */
@NonFinalForTesting
static Clock clock = new SystemClock();
/** Default sleeper for transactions that don't provide one. */
@NonFinalForTesting
static Sleeper sleeper = new SystemSleeper();
/**
* An injected clock that overrides the static clock.
*
* <p>Eventually the static clock should go away when we are 100% injected, but for now we need to
* preserve the old way of overriding the clock in tests by changing the static field.
*/
private final Clock injectedClock;
/** Retry for 8^2 * 100ms = ~25 seconds. */
private static final int NUM_RETRIES = 8;
@Inject
public Ofy(Clock injectedClock) {
this.injectedClock = injectedClock;
}
/**
* Thread local transaction info. There can only be one active transaction on a thread at a given
* time, and this will hold metadata for it.
*/
static final ThreadLocal<TransactionInfo> TRANSACTION_INFO = new ThreadLocal<>();
/** Returns the wrapped Objectify's ObjectifyFactory. */
public ObjectifyFactory factory() {
return ofy().factory();
}
/** Clears the session cache. */
public void clearSessionCache() {
ofy().clear();
}
boolean inTransaction() {
return ofy().getTransaction() != null;
}
public void assertInTransaction() {
checkState(inTransaction(), "Must be called in a transaction");
}
/** Load from Datastore. */
public Loader load() {
return ofy().load();
}
/**
* Delete, augmented to enroll the deleted entities in a commit log.
*
* <p>We only allow this in transactions so commit logs can be written in tandem with the delete.
*/
public Deleter delete() {
return deleteIgnoringReadOnlyWithBackup();
}
/**
* Delete, without any augmentations except to check that we're not saving any virtual entities.
*
* <p>No backups get written.
*/
public Deleter deleteWithoutBackup() {
return deleteIgnoringReadOnlyWithoutBackup();
}
/**
* Save, augmented to enroll the saved entities in a commit log and to check that we're not saving
* virtual entities.
*
* <p>We only allow this in transactions so commit logs can be written in tandem with the save.
*/
public Saver save() {
return saveIgnoringReadOnlyWithBackup();
}
/**
* Save, without any augmentations except to check that we're not saving any virtual entities.
*
* <p>No backups get written.
*/
public Saver saveWithoutBackup() {
return saveIgnoringReadOnlyWithoutBackup();
}
/** Save, ignoring any backups or any read-only settings. */
public Saver saveIgnoringReadOnlyWithoutBackup() {
return new AugmentedSaver() {
@Override
protected void handleSave(Iterable<?> entities) {
checkProhibitedAnnotations(entities, VirtualEntity.class);
}
};
}
/** Delete, ignoring any backups or any read-only settings. */
public Deleter deleteIgnoringReadOnlyWithoutBackup() {
return new AugmentedDeleter() {
@Override
protected void handleDeletion(Iterable<Key<?>> keys) {
checkProhibitedAnnotations(keys, VirtualEntity.class);
}
};
}
/** Save, ignoring any read-only settings (but still write commit logs). */
public Saver saveIgnoringReadOnlyWithBackup() {
return new AugmentedSaver() {
@Override
protected void handleSave(Iterable<?> entities) {
assertInTransaction();
checkState(
Streams.stream(entities).allMatch(Objects::nonNull), "Can't save a null entity.");
checkProhibitedAnnotations(entities, NotBackedUp.class, VirtualEntity.class);
ImmutableMap<Key<?>, ?> keysToEntities = uniqueIndex(entities, Key::create);
TRANSACTION_INFO.get().putSaves(keysToEntities);
}
};
}
/** Delete, ignoring any read-only settings (but still write commit logs). */
public Deleter deleteIgnoringReadOnlyWithBackup() {
return new AugmentedDeleter() {
@Override
protected void handleDeletion(Iterable<Key<?>> keys) {
assertInTransaction();
checkState(Streams.stream(keys).allMatch(Objects::nonNull), "Can't delete a null key.");
checkProhibitedAnnotations(keys, NotBackedUp.class, VirtualEntity.class);
TRANSACTION_INFO.get().putDeletes(keys);
}
};
}
private Clock getClock() {
return injectedClock == null ? clock : injectedClock;
}
/** Execute a transaction. */
<R> R transact(Supplier<R> work) {
// If we are already in a transaction, don't wrap in a CommitLoggedWork.
return inTransaction() ? work.get() : transactNew(work);
}
/**
* Execute a transaction.
*
* <p>This overload is used for transactions that don't return a value, formerly implemented using
* VoidWork.
*/
void transact(Runnable work) {
transact(
() -> {
work.run();
return null;
});
}
/** Pause the current transaction (if any) and complete this one before returning to it. */
public <R> R transactNew(Supplier<R> work) {
// Wrap the Work in a CommitLoggedWork so that we can give transactions a frozen view of time.
return transactCommitLoggedWork(new CommitLoggedWork<>(work, getClock()));
}
/**
* Pause the current transaction (if any) and complete this one before returning to it.
*
* <p>This overload is used for transactions that don't return a value, formerly implemented using
* VoidWork.
*/
void transactNew(Runnable work) {
transactNew(
() -> {
work.run();
return null;
});
}
/**
* Transact with commit logs and retry with exponential backoff.
*
* <p>This method is broken out from {@link #transactNew(Supplier)} for testing purposes.
*/
@VisibleForTesting
<R> R transactCommitLoggedWork(CommitLoggedWork<R> work) {
long baseRetryMillis = getBaseOfyRetryDuration().getMillis();
for (long attempt = 0, sleepMillis = baseRetryMillis;
true;
attempt++, sleepMillis *= 2) {
try {
ofy().transactNew(() -> {
work.run();
return null;
});
return work.getResult();
} catch (TransientFailureException
| DatastoreTimeoutException
| DatastoreFailureException e) {
// TransientFailureExceptions come from task queues and always mean nothing committed.
// TimestampInversionExceptions are thrown by our code and are always retryable as well.
// However, Datastore exceptions might get thrown even if the transaction succeeded.
if ((e instanceof DatastoreTimeoutException || e instanceof DatastoreFailureException)
&& work.hasRun()) {
return work.getResult();
}
if (attempt == NUM_RETRIES) {
throw e; // Give up.
}
sleeper.sleepUninterruptibly(Duration.millis(sleepMillis));
logger.atInfo().withCause(e).log(
"Retrying %s, attempt %d.", e.getClass().getSimpleName(), attempt);
}
}
}
/** A read-only transaction is useful to get strongly consistent reads at a shared timestamp. */
<R> R transactNewReadOnly(Supplier<R> work) {
ReadOnlyWork<R> readOnlyWork = new ReadOnlyWork<>(work, getClock());
try {
ofy().transactNew(() -> {
readOnlyWork.run();
return null;
});
} catch (TransientFailureException | DatastoreTimeoutException | DatastoreFailureException e) {
// These are always retryable for a read-only operation.
return transactNewReadOnly(work);
} catch (KillTransactionException e) {
// Expected; we killed the transaction as a safety measure, and now we can return the result.
return readOnlyWork.getResult();
}
throw new AssertionError(); // How on earth did we get here?
}
void transactNewReadOnly(Runnable work) {
transactNewReadOnly(
() -> {
work.run();
return null;
});
}
/** Execute some work in a transactionless context. */
public <R> R doTransactionless(Supplier<R> work) {
try {
com.googlecode.objectify.ObjectifyService.push(
com.googlecode.objectify.ObjectifyService.ofy().transactionless());
return work.get();
} finally {
com.googlecode.objectify.ObjectifyService.pop();
}
}
/**
* Execute some work with a fresh session cache.
*
* <p>This is useful in cases where we want to load the latest possible data from Datastore but
* don't need point-in-time consistency across loads and consequently don't need a transaction.
* Note that unlike a transaction's fresh session cache, the contents of this cache will be
* discarded once the work completes, rather than being propagated into the enclosing session.
*/
public <R> R doWithFreshSessionCache(Supplier<R> work) {
try {
com.googlecode.objectify.ObjectifyService.push(
com.googlecode.objectify.ObjectifyService.factory().begin());
return work.get();
} finally {
com.googlecode.objectify.ObjectifyService.pop();
}
}
/** Get the time associated with the start of this particular transaction attempt. */
DateTime getTransactionTime() {
assertInTransaction();
return TRANSACTION_INFO.get().transactionTime;
}
/**
* Returns the @Entity-annotated base class for an object that is either an {@code Key<?>} or an
* object of an entity class registered with Objectify.
*/
@VisibleForTesting
static Class<?> getBaseEntityClassFromEntityOrKey(Object entityOrKey) {
// Convert both keys and entities into keys, so that we get consistent behavior in either case.
Key<?> key = (entityOrKey instanceof Key<?> ? (Key<?>) entityOrKey : Key.create(entityOrKey));
// Get the entity class associated with this key's kind, which should be the base @Entity class
// from which the kind name is derived. Don't be tempted to use getMetadata(String kind) or
// getMetadataForEntity(T pojo) instead; the former won't throw an exception for an unknown
// kind (it just returns null) and the latter will return the @EntitySubclass if there is one.
return ofy().factory().getMetadata(key).getEntityClass();
}
/**
* Checks that the base @Entity classes for the provided entities or keys don't have any of the
* specified forbidden annotations.
*/
@SafeVarargs
private static void checkProhibitedAnnotations(
Iterable<?> entitiesOrKeys, Class<? extends Annotation>... annotations) {
for (Object entityOrKey : entitiesOrKeys) {
Class<?> entityClass = getBaseEntityClassFromEntityOrKey(entityOrKey);
for (Class<? extends Annotation> annotation : annotations) {
checkArgument(!entityClass.isAnnotationPresent(annotation),
"Can't save/delete a @%s entity: %s", annotation.getSimpleName(), entityClass);
}
}
}
}
core/src/main/java/google/registry/model/ofy/OfyFilter.java
-44
View File
@@ -1,44 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.ofy;
import google.registry.model.annotations.DeleteAfterMigration;
import java.io.IOException;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
/** A filter that statically registers types with Objectify. */
@DeleteAfterMigration
public class OfyFilter implements Filter {
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain)
throws IOException, ServletException {
filterChain.doFilter(request, response);
}
@Override
public void init(FilterConfig config) {
// Make sure that we've registered all types before we do anything else with Objectify.
ObjectifyService.initOfy();
}
@Override
public void destroy() {}
}
core/src/main/java/google/registry/model/ofy/ReadOnlyWork.java
-42
View File
@@ -1,42 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.ofy;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.util.Clock;
import java.util.function.Supplier;
/** Wrapper for {@link Supplier} that disallows mutations and fails the transaction at the end. */
@DeleteAfterMigration
class ReadOnlyWork<R> extends CommitLoggedWork<R> {
ReadOnlyWork(Supplier<R> work, Clock clock) {
super(work, clock);
}
@Override
protected TransactionInfo createNewTransactionInfo() {
return super.createNewTransactionInfo().setReadOnly();
}
@Override
public void run() {
super.run();
throw new KillTransactionException();
}
/** Exception used to exit a transaction. */
static class KillTransactionException extends RuntimeException {}
}
core/src/main/java/google/registry/model/ofy/RequestCapturingAsyncDatastoreService.java
-194
View File
@@ -1,194 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.ofy;
import static java.util.Collections.synchronizedList;
import com.google.appengine.api.datastore.AsyncDatastoreService;
import com.google.appengine.api.datastore.DatastoreAttributes;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.Index;
import com.google.appengine.api.datastore.Index.IndexState;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.api.datastore.KeyRange;
import com.google.appengine.api.datastore.PreparedQuery;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.Transaction;
import com.google.appengine.api.datastore.TransactionOptions;
import com.google.common.collect.ImmutableList;
import google.registry.model.annotations.DeleteAfterMigration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Future;
/** A proxy for {@link AsyncDatastoreService} that exposes call counts. */
@DeleteAfterMigration
public class RequestCapturingAsyncDatastoreService implements AsyncDatastoreService {
private final AsyncDatastoreService delegate;
// Each outer lists represents Datastore operations, with inner lists representing the keys or
// entities involved in that operation. We use static lists because we care about overall calls to
// Datastore, not calls via a specific instance of the service.
private static List<List<Key>> reads = synchronizedList(new ArrayList<List<Key>>());
private static List<List<Key>> deletes = synchronizedList(new ArrayList<List<Key>>());
private static List<List<Entity>> puts = synchronizedList(new ArrayList<List<Entity>>());
RequestCapturingAsyncDatastoreService(AsyncDatastoreService delegate) {
this.delegate = delegate;
}
public static List<List<Key>> getReads() {
return reads;
}
public static List<List<Key>> getDeletes() {
return deletes;
}
public static List<List<Entity>> getPuts() {
return puts;
}
@Override
public Collection<Transaction> getActiveTransactions() {
return delegate.getActiveTransactions();
}
@Override
public Transaction getCurrentTransaction() {
return delegate.getCurrentTransaction();
}
@Override
public Transaction getCurrentTransaction(Transaction transaction) {
return delegate.getCurrentTransaction(transaction);
}
@Override
public PreparedQuery prepare(Query query) {
return delegate.prepare(query);
}
@Override
public PreparedQuery prepare(Transaction transaction, Query query) {
return delegate.prepare(transaction, query);
}
@Override
public Future<KeyRange> allocateIds(String kind, long num) {
return delegate.allocateIds(kind, num);
}
@Override
public Future<KeyRange> allocateIds(Key parent, String kind, long num) {
return delegate.allocateIds(parent, kind, num);
}
@Override
public Future<Transaction> beginTransaction() {
return delegate.beginTransaction();
}
@Override
public Future<Transaction> beginTransaction(TransactionOptions transaction) {
return delegate.beginTransaction(transaction);
}
@Override
public Future<Void> delete(Key... keys) {
deletes.add(ImmutableList.copyOf(keys));
return delegate.delete(keys);
}
@Override
public Future<Void> delete(Iterable<Key> keys) {
deletes.add(ImmutableList.copyOf(keys));
return delegate.delete(keys);
}
@Override
public Future<Void> delete(Transaction transaction, Key... keys) {
deletes.add(ImmutableList.copyOf(keys));
return delegate.delete(transaction, keys);
}
@Override
public Future<Void> delete(Transaction transaction, Iterable<Key> keys) {
deletes.add(ImmutableList.copyOf(keys));
return delegate.delete(transaction, keys);
}
@Override
public Future<Entity> get(Key key) {
reads.add(ImmutableList.of(key));
return delegate.get(key);
}
@Override
public Future<Map<Key, Entity>> get(Iterable<Key> keys) {
reads.add(ImmutableList.copyOf(keys));
return delegate.get(keys);
}
@Override
public Future<Entity> get(Transaction transaction, Key key) {
reads.add(ImmutableList.of(key));
return delegate.get(transaction, key);
}
@Override
public Future<Map<Key, Entity>> get(Transaction transaction, Iterable<Key> keys) {
reads.add(ImmutableList.copyOf(keys));
return delegate.get(transaction, keys);
}
@Override
public Future<DatastoreAttributes> getDatastoreAttributes() {
return delegate.getDatastoreAttributes();
}
@Override
public Future<Map<Index, IndexState>> getIndexes() {
return delegate.getIndexes();
}
@Override
public Future<Key> put(Entity entity) {
puts.add(ImmutableList.of(entity));
return delegate.put(entity);
}
@Override
public Future<List<Key>> put(Iterable<Entity> entities) {
puts.add(ImmutableList.copyOf(entities));
return delegate.put(entities);
}
@Override
public Future<Key> put(Transaction transaction, Entity entity) {
puts.add(ImmutableList.of(entity));
return delegate.put(transaction, entity);
}
@Override
public Future<List<Key>> put(Transaction transaction, Iterable<Entity> entities) {
puts.add(ImmutableList.copyOf(entities));
return delegate.put(transaction, entities);
}
}
core/src/main/java/google/registry/model/ofy/TransactionInfo.java
-73
View File
@@ -1,73 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.model.ofy;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Maps.toMap;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.googlecode.objectify.Key;
import google.registry.model.annotations.DeleteAfterMigration;
import java.util.Map;
import org.joda.time.DateTime;
/** Metadata for an {@link Ofy} transaction that saves commit logs. */
@DeleteAfterMigration
public class TransactionInfo {
@VisibleForTesting
public enum Delete {
SENTINEL
}
/** Logical "now" of the transaction. */
DateTime transactionTime;
/** Whether this is a read-only transaction. */
private boolean readOnly;
/**
* Accumulator of save/delete operations performed in transaction.
*
* <p>The {@link ImmutableMap} builder provides us the benefit of not permitting duplicates.
* This allows us to avoid potential race conditions where the same key is mutated twice in a
* transaction.
*/
private final ImmutableMap.Builder<Key<?>, Object> changesBuilder = new ImmutableMap.Builder<>();
TransactionInfo(DateTime now) {
this.transactionTime = now;
}
TransactionInfo setReadOnly() {
this.readOnly = true;
return this;
}
void assertNotReadOnly() {
checkState(!readOnly, "This is a read only transaction.");
}
void putSaves(Map<Key<?>, ?> keysToEntities) {
assertNotReadOnly();
changesBuilder.putAll(keysToEntities);
}
void putDeletes(Iterable<Key<?>> keys) {
assertNotReadOnly();
changesBuilder.putAll(toMap(keys, k -> Delete.SENTINEL));
}
}
core/src/main/java/google/registry/persistence/VKey.java
+1 -1
View File
@@ -75,7 +75,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
* Constructs a {@link VKey} for an {@link EppResource } from the string representation.
*
* <p>The string representation is obtained from the {@link #stringify()} function and like this:
* {@code kind:TestObject@sql:rO0ABXQAA2Zvbw}
* {@code kind:SomeEntity@sql:rO0ABXQAA2Zvbw}
*/
public static <T extends EppResource> VKey<T> createEppVKeyFromString(String keyString) {
ImmutableMap<String, String> kvs =
core/src/main/java/google/registry/persistence/transaction/QueryComposer.java
-8
View File
@@ -149,14 +149,6 @@ public abstract class QueryComposer<T> {
/**
* Enum used to specify comparison operations, e.g. {@code where("fieldName", Comparator.NE,
* "someval")'}.
*
* <p>These contain values that specify the comparison behavior for both objectify and criteria
* queries. For objectify, we provide a string to be appended to the field name in a {@code
* filter()} expression. For criteria queries we provide a function that knows how to obtain a
* {@link WhereOperator} from a {@link CriteriaBuilder}.
*
* <p>Note that the objectify strings for comparators other than equality are preceded by a space
* because {@code filter()} expects the fieldname to be separated from the operator by a space.
*/
public enum Comparator {
/**
core/src/main/java/google/registry/tmch/TmchCertificateAuthority.java
+2 -2
View File
@@ -19,12 +19,12 @@ import static google.registry.config.RegistryConfig.ConfigModule.TmchCaMode.PROD
import static google.registry.config.RegistryConfig.getSingletonCacheRefreshDuration;
import static google.registry.util.ResourceUtils.readResourceUtf8;
import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.google.common.collect.ImmutableMap;
import google.registry.config.RegistryConfig.Config;
import google.registry.config.RegistryConfig.ConfigModule.TmchCaMode;
import google.registry.model.CacheUtils;
import google.registry.model.CacheUtils.AppEngineEnvironmentCacheLoader;
import google.registry.model.tmch.TmchCrl;
import google.registry.util.Clock;
import google.registry.util.X509Utils;
@@ -78,7 +78,7 @@ public final class TmchCertificateAuthority {
private static final LoadingCache<TmchCaMode, X509CRL> CRL_CACHE =
CacheUtils.newCacheBuilder(getSingletonCacheRefreshDuration())
.build(
new AppEngineEnvironmentCacheLoader<TmchCaMode, X509CRL>() {
new CacheLoader<TmchCaMode, X509CRL>() {
@Override
public X509CRL load(final TmchCaMode tmchCaMode) throws GeneralSecurityException {
Optional<TmchCrl> storedCrl = TmchCrl.get();
core/src/main/java/google/registry/tools/EntityWrapper.java
-80
View File
@@ -1,80 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools;
import com.google.appengine.api.datastore.Entity;
import com.google.auto.value.AutoValue;
import com.google.common.base.Objects;
/**
* Wraps {@link Entity} for ease of processing in collections.
*
* <p>Note that the {@link #hashCode}/{@link #equals} methods are based on both the entity's key and
* its properties.
*/
final class EntityWrapper {
private static final String TEST_ENTITY_KIND = "TestEntity";
private final Entity entity;
EntityWrapper(Entity entity) {
this.entity = entity;
}
public Entity getEntity() {
return entity;
}
@Override
public boolean equals(Object that) {
if (that instanceof EntityWrapper) {
EntityWrapper thatEntity = (EntityWrapper) that;
return entity.equals(thatEntity.entity)
&& entity.getProperties().equals(thatEntity.entity.getProperties());
}
return false;
}
@Override
public int hashCode() {
return Objects.hashCode(entity.getKey(), entity.getProperties());
}
@Override
public String toString() {
return "EntityWrapper(" + entity + ")";
}
public static EntityWrapper from(int id, Property... properties) {
Entity entity = new Entity(TEST_ENTITY_KIND, id);
for (Property prop : properties) {
entity.setProperty(prop.name(), prop.value());
}
return new EntityWrapper(entity);
}
@AutoValue
abstract static class Property {
static Property create(String name, Object value) {
return new AutoValue_EntityWrapper_Property(name, value);
}
abstract String name();
abstract Object value();
}
}
core/src/main/java/google/registry/tools/RegistryCli.java
-10
View File
@@ -29,7 +29,6 @@ import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import google.registry.config.RegistryConfig;
import google.registry.model.ofy.ObjectifyService;
import google.registry.persistence.transaction.TransactionManagerFactory;
import google.registry.tools.AuthModule.LoginRequiredException;
import google.registry.tools.params.ParameterFactory;
@@ -256,15 +255,6 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
}
installer.install(options);
// Database setup -- we also only ever do this if "installer" is null, just so that it's
// only done once.
// Ensure that all entity classes are loaded before command code runs.
ObjectifyService.initOfy();
// Make sure we start the command with a clean cache, so that any previous command won't
// interfere with this one.
ObjectifyService.auditedOfy().clearSessionCache();
// Enable Cloud SQL for command that needs remote API as they will very likely use
// Cloud SQL after the database migration. Note that the DB password is stored in Datastore
// and it is already initialized above.
core/src/main/java/google/registry/tools/javascrap/CreateSyntheticDomainHistoriesCommand.java
-2
View File
@@ -27,7 +27,6 @@ import google.registry.config.CredentialModule;
import google.registry.config.RegistryConfig;
import google.registry.config.RegistryConfig.Config;
import google.registry.model.domain.Domain;
import google.registry.model.ofy.ObjectifyService;
import google.registry.model.reporting.HistoryEntry;
import google.registry.persistence.VKey;
import google.registry.tools.CommandWithConnection;
@@ -199,7 +198,6 @@ public class CreateSyntheticDomainHistoriesCommand extends ConfirmingCommand
} catch (IOException e) {
throw new RuntimeException(e);
}
ObjectifyService.initOfy();
return installer;
}
core/src/main/javascript/soyutils_usegoog.js
-5
View File
@@ -928,7 +928,6 @@ soy.$$cleanHtml = function(value, opt_safeTags) {
};
// LINT.IfChange(htmlToText)
/**
* Converts HTML to plain text by removing tags, normalizing spaces and
* converting entities.
@@ -1008,10 +1007,6 @@ soy.$$htmlToText = function(value) {
/** @private @const */
soy.BLOCK_TAGS_RE_ =
/^\/?(address|blockquote|dd|div|dl|dt|h[1-6]|hr|li|ol|p|pre|table|tr|ul)$/i;
// LINT.ThenChange(
// ../../../third_party/java_src/soy/java/com/google/template/soy/basicfunctions/HtmlToText.java,
// ../../../third_party/java_src/soy/python/runtime/sanitize.py:htmlToText)
/**
* Escapes HTML, except preserves entities.