mirror of
https://github.com/google/nomulus
synced 2026-02-05 04:21:07 +00:00
Daggerize/Actionize the load snapshot servlet
This is needed to eliminate the last non-injected use of BigqueryFactory. It gets rid of the deprecated HttpServletUtils now that we don't have lots of separate servlets, and changes the check snapshot servlet's usage of such. It also fixes up some related minor style issues in the update snapshot action (and its injectable parameters). ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=120547998
This commit is contained in:
mcilwain
Justine Tunney
@@ -15,8 +15,11 @@
|
||||
package com.google.domain.registry.export;
|
||||
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static com.google.common.collect.Sets.intersection;
|
||||
import static com.google.common.html.HtmlEscapers.htmlEscaper;
|
||||
import static com.google.domain.registry.util.HttpServletUtils.getRequiredParameterValue;
|
||||
import static com.google.domain.registry.export.LoadSnapshotAction.enqueueLoadSnapshotTask;
|
||||
import static com.google.domain.registry.request.RequestParameters.extractRequiredParameter;
|
||||
import static com.google.domain.registry.util.FormattingLogger.getLoggerForCallerClass;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
@@ -28,12 +31,12 @@ import com.google.appengine.api.taskqueue.TaskHandle;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Optional;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.net.MediaType;
|
||||
import com.google.domain.registry.export.DatastoreBackupInfo.BackupStatus;
|
||||
import com.google.domain.registry.request.HttpException.BadRequestException;
|
||||
import com.google.domain.registry.util.FormattingLogger;
|
||||
import com.google.domain.registry.util.NonFinalForTesting;
|
||||
|
||||
@@ -42,6 +45,7 @@ import org.joda.time.PeriodType;
|
||||
import org.joda.time.format.PeriodFormat;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
@@ -62,14 +66,11 @@ public class CheckSnapshotServlet extends HttpServlet {
|
||||
/** The maximum amount of time we allow a backup to run before abandoning it. */
|
||||
static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20);
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static DatastoreBackupService backupService = DatastoreBackupService.get();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static LoadSnapshotServlet loadSnapshotServlet = new LoadSnapshotServlet();
|
||||
|
||||
@Override
|
||||
public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
try {
|
||||
@@ -87,18 +88,27 @@ public class CheckSnapshotServlet extends HttpServlet {
|
||||
|
||||
@Override
|
||||
public void doGet(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
String snapshotName = getRequiredParameterValue(req, SNAPSHOT_NAME_PARAM);
|
||||
rsp.getWriter().write(backupService.findByName(snapshotName).getInformation());
|
||||
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
|
||||
try {
|
||||
String snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
|
||||
rsp.getWriter().write(backupService.findByName(snapshotName).getInformation());
|
||||
} catch (BadRequestException e) {
|
||||
throw new IllegalArgumentException(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
String snapshotName = getRequiredParameterValue(req, SNAPSHOT_NAME_PARAM);
|
||||
// TODO(b/19237926): make this non-optional once all new tasks will have this parameter.
|
||||
String kindsToLoadParam = req.getParameter(SNAPSHOT_KINDS_TO_LOAD_PARAM);
|
||||
Optional<ImmutableSet<String>> kindsToLoad = Optional.fromNullable(
|
||||
kindsToLoadParam == null ? null
|
||||
: ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam)));
|
||||
String snapshotName;
|
||||
String kindsToLoadParam;
|
||||
// TODO(b/28266757): Remove this try/catch/rethrow block once this servlet is Daggerized.
|
||||
try {
|
||||
snapshotName = extractRequiredParameter(req, SNAPSHOT_NAME_PARAM);
|
||||
kindsToLoadParam = extractRequiredParameter(req, SNAPSHOT_KINDS_TO_LOAD_PARAM);
|
||||
} catch (BadRequestException e) {
|
||||
throw new IllegalArgumentException(e.getMessage());
|
||||
}
|
||||
Set<String> kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam));
|
||||
|
||||
// Look up the backup by the provided name, stopping if we can't find it.
|
||||
DatastoreBackupInfo backup;
|
||||
@@ -137,21 +147,20 @@ public class CheckSnapshotServlet extends HttpServlet {
|
||||
String snapshotId = snapshotName.startsWith(ExportSnapshotServlet.SNAPSHOT_PREFIX)
|
||||
? snapshotName.substring(ExportSnapshotServlet.SNAPSHOT_PREFIX.length())
|
||||
: backup.getStartTime().toString("YYYYMMdd_HHmmss");
|
||||
// Log a warning if kindsToLoad is specified and not a subset of the exported snapshot kinds.
|
||||
if (kindsToLoad.isPresent() && !backup.getKinds().containsAll(kindsToLoad.get())) {
|
||||
logger.warningfmt("Kinds to load included non-exported kinds: %s",
|
||||
Sets.difference(kindsToLoad.get(), backup.getKinds()));
|
||||
// Log a warning if kindsToLoad is not a subset of the exported snapshot kinds.
|
||||
if (!backup.getKinds().containsAll(kindsToLoad)) {
|
||||
logger.warningfmt(
|
||||
"Kinds to load included non-exported kinds: %s",
|
||||
Sets.difference(kindsToLoad, backup.getKinds()));
|
||||
}
|
||||
// Load kinds from the snapshot, limited to those also in kindsToLoad (if it's present).
|
||||
ImmutableSet<String> exportedKindsToLoad = ImmutableSet.copyOf(kindsToLoad.isPresent()
|
||||
? Sets.intersection(backup.getKinds(), kindsToLoad.get())
|
||||
: backup.getKinds());
|
||||
ImmutableSet<String> exportedKindsToLoad =
|
||||
ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad));
|
||||
String message = String.format("Datastore backup %s complete - ", snapshotName);
|
||||
if (exportedKindsToLoad.isEmpty()) {
|
||||
message += "no kinds to load into BigQuery";
|
||||
} else {
|
||||
loadSnapshotServlet.enqueueLoadTask(
|
||||
snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
|
||||
enqueueLoadSnapshotTask(snapshotId, backup.getGcsFilename().get(), exportedKindsToLoad);
|
||||
message += "BigQuery load task enqueued";
|
||||
}
|
||||
logger.info(message);
|
||||
|
||||
@@ -17,9 +17,12 @@ package com.google.domain.registry.export;
|
||||
import static com.google.domain.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
|
||||
import static com.google.domain.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
|
||||
import static com.google.domain.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.SNAPSHOT_DATASET_ID_PARAM;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.SNAPSHOT_KIND_PARAM;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.SNAPSHOT_TABLE_ID_PARAM;
|
||||
import static com.google.domain.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_FILE_PARAM;
|
||||
import static com.google.domain.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_ID_PARAM;
|
||||
import static com.google.domain.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARAM;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM;
|
||||
import static com.google.domain.registry.request.RequestParameters.extractRequiredHeader;
|
||||
|
||||
import com.google.domain.registry.request.Header;
|
||||
@@ -35,21 +38,39 @@ import javax.servlet.http.HttpServletRequest;
|
||||
public final class ExportRequestModule {
|
||||
|
||||
@Provides
|
||||
@Parameter(SNAPSHOT_DATASET_ID_PARAM)
|
||||
static String provideDatasetId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, SNAPSHOT_DATASET_ID_PARAM);
|
||||
@Parameter(UPDATE_SNAPSHOT_DATASET_ID_PARAM)
|
||||
static String provideUpdateSnapshotDatasetId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, UPDATE_SNAPSHOT_DATASET_ID_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(SNAPSHOT_TABLE_ID_PARAM)
|
||||
static String provideTableId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, SNAPSHOT_TABLE_ID_PARAM);
|
||||
@Parameter(UPDATE_SNAPSHOT_TABLE_ID_PARAM)
|
||||
static String provideUpdateSnapshotTableId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, UPDATE_SNAPSHOT_TABLE_ID_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(SNAPSHOT_KIND_PARAM)
|
||||
static String provideKind(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, SNAPSHOT_KIND_PARAM);
|
||||
@Parameter(UPDATE_SNAPSHOT_KIND_PARAM)
|
||||
static String provideUpdateSnapshotKind(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, UPDATE_SNAPSHOT_KIND_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(LOAD_SNAPSHOT_FILE_PARAM)
|
||||
static String provideLoadSnapshotFile(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, LOAD_SNAPSHOT_FILE_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(LOAD_SNAPSHOT_ID_PARAM)
|
||||
static String provideLoadSnapshotId(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, LOAD_SNAPSHOT_ID_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Parameter(LOAD_SNAPSHOT_KINDS_PARAM)
|
||||
static String provideLoadSnapshotKinds(HttpServletRequest req) {
|
||||
return extractRequiredHeader(req, LOAD_SNAPSHOT_KINDS_PARAM);
|
||||
}
|
||||
|
||||
@Provides
|
||||
|
||||
@@ -14,27 +14,19 @@
|
||||
|
||||
package com.google.domain.registry.export;
|
||||
|
||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static com.google.common.base.MoreObjects.firstNonNull;
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.html.HtmlEscapers.htmlEscaper;
|
||||
import static com.google.domain.registry.util.HttpServletUtils.getRequiredParameterValue;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static com.google.domain.registry.export.UpdateSnapshotViewAction.createViewUpdateTask;
|
||||
import static com.google.domain.registry.request.Action.Method.POST;
|
||||
import static com.google.domain.registry.util.FormattingLogger.getLoggerForCallerClass;
|
||||
|
||||
import com.google.api.client.extensions.appengine.http.UrlFetchTransport;
|
||||
import com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential;
|
||||
import com.google.api.client.json.jackson2.JacksonFactory;
|
||||
import com.google.api.services.bigquery.Bigquery;
|
||||
import com.google.api.services.bigquery.BigqueryScopes;
|
||||
import com.google.api.services.bigquery.model.Dataset;
|
||||
import com.google.api.services.bigquery.model.DatasetReference;
|
||||
import com.google.api.services.bigquery.model.Job;
|
||||
import com.google.api.services.bigquery.model.JobConfiguration;
|
||||
import com.google.api.services.bigquery.model.JobConfigurationLoad;
|
||||
import com.google.api.services.bigquery.model.JobReference;
|
||||
import com.google.api.services.bigquery.model.TableReference;
|
||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
||||
import com.google.appengine.api.taskqueue.TaskHandle;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
@@ -42,99 +34,82 @@ import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.net.MediaType;
|
||||
import com.google.domain.registry.bigquery.BigqueryFactory;
|
||||
import com.google.domain.registry.bigquery.BigqueryJobFailureException;
|
||||
import com.google.domain.registry.bigquery.BigqueryUtils.SourceFormat;
|
||||
import com.google.domain.registry.bigquery.BigqueryUtils.WriteDisposition;
|
||||
import com.google.domain.registry.config.RegistryEnvironment;
|
||||
import com.google.domain.registry.config.ConfigModule.Config;
|
||||
import com.google.domain.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer;
|
||||
import com.google.domain.registry.request.Action;
|
||||
import com.google.domain.registry.request.HttpException.BadRequestException;
|
||||
import com.google.domain.registry.request.HttpException.InternalServerErrorException;
|
||||
import com.google.domain.registry.request.Parameter;
|
||||
import com.google.domain.registry.util.Clock;
|
||||
import com.google.domain.registry.util.FormattingLogger;
|
||||
import com.google.domain.registry.util.NonFinalForTesting;
|
||||
import com.google.domain.registry.util.Retrier;
|
||||
import com.google.domain.registry.util.SystemClock;
|
||||
import com.google.domain.registry.util.SystemSleeper;
|
||||
import com.google.domain.registry.util.TaskEnqueuer;
|
||||
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/** Load a datastore snapshot from Google Cloud Storage into BigQuery. */
|
||||
public class LoadSnapshotServlet extends HttpServlet {
|
||||
|
||||
private static final RegistryEnvironment ENVIRONMENT = RegistryEnvironment.get();
|
||||
/** Action to load a Datastore snapshot from Google Cloud Storage into BigQuery. */
|
||||
@Action(path = LoadSnapshotAction.PATH, method = POST)
|
||||
public class LoadSnapshotAction implements Runnable {
|
||||
|
||||
/** Parameter names for passing parameters into the servlet. */
|
||||
static final String SNAPSHOT_ID_PARAM = "id";
|
||||
static final String SNAPSHOT_FILE_PARAM = "file";
|
||||
static final String SNAPSHOT_KINDS_PARAM = "kinds";
|
||||
static final String LOAD_SNAPSHOT_ID_PARAM = "id";
|
||||
static final String LOAD_SNAPSHOT_FILE_PARAM = "file";
|
||||
static final String LOAD_SNAPSHOT_KINDS_PARAM = "kinds";
|
||||
|
||||
static final String SNAPSHOTS_DATASET = "snapshots";
|
||||
|
||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
||||
static final String QUEUE = "export-snapshot"; // See queue.xml.
|
||||
static final String PATH = "/_dr/task/loadSnapshot"; // See web.xml.
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static Clock clock = new SystemClock();
|
||||
@Inject BigqueryFactory bigqueryFactory;
|
||||
@Inject BigqueryPollJobEnqueuer bigqueryPollEnqueuer;
|
||||
@Inject Clock clock;
|
||||
@Inject @Config("projectId") String projectId;
|
||||
@Inject @Parameter(LOAD_SNAPSHOT_FILE_PARAM) String snapshotFile;
|
||||
@Inject @Parameter(LOAD_SNAPSHOT_ID_PARAM) String snapshotId;
|
||||
@Inject @Parameter(LOAD_SNAPSHOT_KINDS_PARAM) String snapshotKinds;
|
||||
@Inject LoadSnapshotAction() {}
|
||||
|
||||
@NonFinalForTesting
|
||||
private static BigqueryFactory bigqueryFactory = new BigqueryFactory();
|
||||
|
||||
@NonFinalForTesting
|
||||
private static BigqueryPollJobEnqueuer bigqueryPollEnqueuer = new BigqueryPollJobEnqueuer(
|
||||
new TaskEnqueuer(new Retrier(new SystemSleeper(), 5)));
|
||||
|
||||
/** Enqueue a task for starting a backup load. Not static for better testability. */
|
||||
public TaskHandle enqueueLoadTask(
|
||||
/** Enqueue a task for starting a backup load. */
|
||||
public static TaskHandle enqueueLoadSnapshotTask(
|
||||
String snapshotId, String gcsFile, ImmutableSet<String> kinds) {
|
||||
return QueueFactory.getQueue(QUEUE).add(
|
||||
return getQueue(QUEUE).add(
|
||||
TaskOptions.Builder.withUrl(PATH)
|
||||
.method(Method.POST)
|
||||
.param(SNAPSHOT_ID_PARAM, snapshotId)
|
||||
.param(SNAPSHOT_FILE_PARAM, gcsFile)
|
||||
.param(SNAPSHOT_KINDS_PARAM, Joiner.on(',').join(kinds)));
|
||||
.param(LOAD_SNAPSHOT_ID_PARAM, snapshotId)
|
||||
.param(LOAD_SNAPSHOT_FILE_PARAM, gcsFile)
|
||||
.param(LOAD_SNAPSHOT_KINDS_PARAM, Joiner.on(',').join(kinds)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doPost(HttpServletRequest req, HttpServletResponse rsp) throws IOException {
|
||||
public void run() {
|
||||
try {
|
||||
String snapshotId = getRequiredParameterValue(req, SNAPSHOT_ID_PARAM);
|
||||
String gcsFilename = getRequiredParameterValue(req, SNAPSHOT_FILE_PARAM);
|
||||
ImmutableList<String> kinds = ImmutableList.copyOf(
|
||||
Splitter.on(',').splitToList(getRequiredParameterValue(req, SNAPSHOT_KINDS_PARAM)));
|
||||
|
||||
String message = loadSnapshot(snapshotId, gcsFilename, kinds);
|
||||
|
||||
rsp.setStatus(SC_OK);
|
||||
rsp.setContentType(MediaType.PLAIN_TEXT_UTF_8.toString());
|
||||
rsp.getWriter().write("OK\n\n" + message);
|
||||
String message =
|
||||
loadSnapshot(snapshotId, snapshotFile, Splitter.on(',').split(snapshotKinds));
|
||||
logger.infofmt("Loaded snapshot successfully: %s", message);
|
||||
} catch (Throwable e) {
|
||||
logger.severe(e, e.toString());
|
||||
rsp.sendError(
|
||||
e instanceof IllegalArgumentException ? SC_BAD_REQUEST : SC_INTERNAL_SERVER_ERROR,
|
||||
htmlEscaper().escape(firstNonNull(e.getMessage(), e.toString())));
|
||||
logger.severe(e, "Error loading snapshot");
|
||||
if (e instanceof IllegalArgumentException) {
|
||||
throw new BadRequestException("Error calling load snapshot: " + e.getMessage(), e);
|
||||
} else {
|
||||
throw new InternalServerErrorException(
|
||||
"Error loading snapshot: " + firstNonNull(e.getMessage(), e.toString()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String loadSnapshot(String snapshotId, String gcsFilename, ImmutableList<String> kinds)
|
||||
private String loadSnapshot(String snapshotId, String gcsFilename, Iterable<String> kinds)
|
||||
throws IOException {
|
||||
Bigquery bigquery = bigqueryFactory.create(
|
||||
getClass().getSimpleName(),
|
||||
new UrlFetchTransport(),
|
||||
new JacksonFactory(),
|
||||
new AppIdentityCredential(BigqueryScopes.all()));
|
||||
String projectId = ENVIRONMENT.config().getProjectId();
|
||||
Bigquery bigquery = bigqueryFactory.create(projectId, SNAPSHOTS_DATASET);
|
||||
DateTime now = clock.nowUtc();
|
||||
|
||||
ensureDataset(bigquery, projectId, ENVIRONMENT.config().getSnapshotsDataset());
|
||||
|
||||
String loadMessage =
|
||||
String.format("Loading datastore snapshot %s from %s...", snapshotId, gcsFilename);
|
||||
logger.info(loadMessage);
|
||||
@@ -155,9 +130,8 @@ public class LoadSnapshotServlet extends HttpServlet {
|
||||
// well-known view in BigQuery to point at the newly loaded snapshot table for this kind.
|
||||
bigqueryPollEnqueuer.enqueuePollTask(
|
||||
jobRef,
|
||||
UpdateSnapshotViewAction.createViewUpdateTask(
|
||||
ENVIRONMENT.config().getSnapshotsDataset(), tableId, kindName),
|
||||
QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
|
||||
createViewUpdateTask(SNAPSHOTS_DATASET, tableId, kindName),
|
||||
getQueue(UpdateSnapshotViewAction.QUEUE));
|
||||
|
||||
builder.append(String.format(" - %s:%s\n", projectId, jobId));
|
||||
logger.infofmt("Submitted load job %s:%s", projectId, jobId);
|
||||
@@ -165,24 +139,6 @@ public class LoadSnapshotServlet extends HttpServlet {
|
||||
return builder.toString();
|
||||
}
|
||||
|
||||
private static void ensureDataset(Bigquery bigquery, String projectId, String datasetId)
|
||||
throws IOException {
|
||||
try {
|
||||
bigquery.datasets()
|
||||
.insert(projectId,
|
||||
new Dataset().setDatasetReference(
|
||||
new DatasetReference()
|
||||
.setProjectId(projectId)
|
||||
.setDatasetId(datasetId)))
|
||||
.execute();
|
||||
} catch (IOException e) {
|
||||
// Swallow errors about a duplicate dataset, and throw any other ones.
|
||||
if (!BigqueryJobFailureException.create(e).getReason().equals("duplicate")) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static String getBackupInfoFileForKind(String backupInfoFile, String kindName) {
|
||||
String extension = ".backup_info";
|
||||
checkArgument(backupInfoFile.endsWith(extension), "backup info file extension missing");
|
||||
@@ -190,10 +146,10 @@ public class LoadSnapshotServlet extends HttpServlet {
|
||||
return Joiner.on('.').join(prefix, kindName, extension.substring(1));
|
||||
}
|
||||
|
||||
private static Job makeLoadJob(JobReference jobRef, String sourceUri, String tableId) {
|
||||
private Job makeLoadJob(JobReference jobRef, String sourceUri, String tableId) {
|
||||
TableReference tableReference = new TableReference()
|
||||
.setProjectId(jobRef.getProjectId())
|
||||
.setDatasetId(ENVIRONMENT.config().getSnapshotsDataset())
|
||||
.setDatasetId(SNAPSHOTS_DATASET)
|
||||
.setTableId(tableId);
|
||||
return new Job()
|
||||
.setJobReference(jobRef)
|
||||
@@ -23,7 +23,7 @@ import com.google.api.services.bigquery.model.ViewDefinition;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions;
|
||||
import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
||||
import com.google.domain.registry.bigquery.BigqueryFactory;
|
||||
import com.google.domain.registry.config.RegistryEnvironment;
|
||||
import com.google.domain.registry.config.ConfigModule.Config;
|
||||
import com.google.domain.registry.request.Action;
|
||||
import com.google.domain.registry.request.HttpException.InternalServerErrorException;
|
||||
import com.google.domain.registry.request.Parameter;
|
||||
@@ -38,12 +38,12 @@ import javax.inject.Inject;
|
||||
@Action(path = UpdateSnapshotViewAction.PATH, method = POST)
|
||||
public class UpdateSnapshotViewAction implements Runnable {
|
||||
|
||||
private static final RegistryEnvironment ENVIRONMENT = RegistryEnvironment.get();
|
||||
|
||||
/** Headers for passing parameters into the servlet. */
|
||||
static final String SNAPSHOT_DATASET_ID_PARAM = "dataset";
|
||||
static final String SNAPSHOT_TABLE_ID_PARAM = "table";
|
||||
static final String SNAPSHOT_KIND_PARAM = "kind";
|
||||
static final String UPDATE_SNAPSHOT_DATASET_ID_PARAM = "dataset";
|
||||
static final String UPDATE_SNAPSHOT_TABLE_ID_PARAM = "table";
|
||||
static final String UPDATE_SNAPSHOT_KIND_PARAM = "kind";
|
||||
|
||||
static final String LATEST_SNAPSHOT_DATASET = "latest_snapshot";
|
||||
|
||||
/** Servlet-specific details needed for enqueuing tasks against itself. */
|
||||
static final String QUEUE = "export-snapshot-update-view"; // See queue.xml.
|
||||
@@ -51,9 +51,10 @@ public class UpdateSnapshotViewAction implements Runnable {
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
|
||||
@Inject @Parameter(SNAPSHOT_DATASET_ID_PARAM) String datasetId;
|
||||
@Inject @Parameter(SNAPSHOT_TABLE_ID_PARAM) String tableId;
|
||||
@Inject @Parameter(SNAPSHOT_KIND_PARAM) String kindName;
|
||||
@Inject @Parameter(UPDATE_SNAPSHOT_DATASET_ID_PARAM) String datasetId;
|
||||
@Inject @Parameter(UPDATE_SNAPSHOT_TABLE_ID_PARAM) String tableId;
|
||||
@Inject @Parameter(UPDATE_SNAPSHOT_KIND_PARAM) String kindName;
|
||||
@Inject @Config("projectId") String projectId;
|
||||
@Inject BigqueryFactory bigqueryFactory;
|
||||
@Inject UpdateSnapshotViewAction() {}
|
||||
|
||||
@@ -62,9 +63,9 @@ public class UpdateSnapshotViewAction implements Runnable {
|
||||
String datasetId, String tableId, String kindName) {
|
||||
return TaskOptions.Builder.withUrl(PATH)
|
||||
.method(Method.POST)
|
||||
.param(SNAPSHOT_DATASET_ID_PARAM, datasetId)
|
||||
.param(SNAPSHOT_TABLE_ID_PARAM, tableId)
|
||||
.param(SNAPSHOT_KIND_PARAM, kindName);
|
||||
.param(UPDATE_SNAPSHOT_DATASET_ID_PARAM, datasetId)
|
||||
.param(UPDATE_SNAPSHOT_TABLE_ID_PARAM, tableId)
|
||||
.param(UPDATE_SNAPSHOT_KIND_PARAM, kindName);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -79,14 +80,12 @@ public class UpdateSnapshotViewAction implements Runnable {
|
||||
|
||||
private void updateSnapshotView(String datasetId, String tableId, String kindName)
|
||||
throws IOException {
|
||||
String projectId = ENVIRONMENT.config().getProjectId();
|
||||
Bigquery bigquery =
|
||||
bigqueryFactory.create(projectId, ENVIRONMENT.config().getLatestSnapshotDataset());
|
||||
Bigquery bigquery = bigqueryFactory.create(projectId, LATEST_SNAPSHOT_DATASET);
|
||||
|
||||
updateTable(bigquery, new Table()
|
||||
.setTableReference(new TableReference()
|
||||
.setProjectId(projectId)
|
||||
.setDatasetId(ENVIRONMENT.config().getLatestSnapshotDataset())
|
||||
.setDatasetId(LATEST_SNAPSHOT_DATASET)
|
||||
.setTableId(kindName))
|
||||
.setView(new ViewDefinition().setQuery(
|
||||
SqlTemplate.create("SELECT * FROM [%DATASET%.%TABLE%]")
|
||||
@@ -94,10 +93,12 @@ public class UpdateSnapshotViewAction implements Runnable {
|
||||
.put("TABLE", tableId)
|
||||
.build())));
|
||||
|
||||
String message = String.format(
|
||||
logger.infofmt(
|
||||
"Updated view %s:%s to point at snapshot table %s:%s.",
|
||||
ENVIRONMENT.config().getLatestSnapshotDataset(), kindName, datasetId, tableId);
|
||||
logger.info(message);
|
||||
LATEST_SNAPSHOT_DATASET,
|
||||
kindName,
|
||||
datasetId,
|
||||
tableId);
|
||||
}
|
||||
|
||||
private static void updateTable(Bigquery bigquery, Table table) throws IOException {
|
||||
|
||||
Reference in New Issue
Block a user