mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-05-12 19:01:49 +00:00
Code Issues Releases Wiki Activity

Migrate to new Sheets v4 api

Browse Source 
This moves us from the oudated google/data XML api to the OnePlatform REST/JSON api, finally silencing the deprecation warnings we've been seeing.

The synchronization algorithm diffs the spreadsheet's current values with its internally sourced values, adding the row to a batch update request if there's a discrepancy. Additional internal data are added as an append operation to the end of the sheet, and any extraneous spreadsheet data is cleared from the spreadsheet.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=169273590
This commit is contained in:
larryruili
2017-09-19 11:39:35 -07:00
committed by jianglai
parent 67116c5fa1
commit 578673141c
9 changed files with 305 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
java/google/registry/export/sheet/BUILD
View File

@@ -8,7 +8,6 @@ java_library(
name = "sheet",
srcs = glob(["*.java"]),
deps = [
"//java/com/google/gdata:spreadsheet",
"//java/google/registry/config",
"//java/google/registry/model",
"//java/google/registry/request",
@@ -17,11 +16,12 @@ java_library(
"//java/google/registry/util",
"//third_party/java/objectify:objectify-v4_1",
"@com_google_api_client",
"@com_google_apis_google_api_services_sheets",
"@com_google_appengine_api_1_0_sdk",
"@com_google_code_findbugs_jsr305",
"@com_google_dagger",
"@com_google_gdata_core",
"@com_google_guava",
"@com_google_http_client",
"@javax_servlet_api",
"@joda_time",
],

192
java/google/registry/export/sheet/SheetSynchronizer.java
View File

@@ -14,31 +14,32 @@
package google.registry.export.sheet;
import static com.google.common.base.Strings.nullToEmpty;
import com.google.api.services.sheets.v4.Sheets;
import com.google.api.services.sheets.v4.model.AppendValuesResponse;
import com.google.api.services.sheets.v4.model.BatchUpdateValuesRequest;
import com.google.api.services.sheets.v4.model.BatchUpdateValuesResponse;
import com.google.api.services.sheets.v4.model.ClearValuesRequest;
import com.google.api.services.sheets.v4.model.ClearValuesResponse;
import com.google.api.services.sheets.v4.model.ValueRange;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
import com.google.gdata.data.spreadsheet.CustomElementCollection;
import com.google.gdata.data.spreadsheet.ListEntry;
import com.google.gdata.data.spreadsheet.ListFeed;
import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
import com.google.gdata.data.spreadsheet.WorksheetEntry;
import com.google.gdata.util.ServiceException;
import google.registry.util.Retrier;
import google.registry.util.FormattingLogger;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import javax.inject.Inject;
/** Generic data synchronization utility for Google Spreadsheets. */
class SheetSynchronizer {
private static final String SPREADSHEET_URL_PREFIX =
"https://spreadsheets.google.com/feeds/spreadsheets/";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject SpreadsheetService spreadsheetService;
private static final String SHEET_NAME = "Registrars";
@Inject Sheets sheetsService;
@Inject SheetSynchronizer() {}
@Inject Retrier retrier;
/**
* Replace the contents of a Google Spreadsheet with {@code data}.
@@ -50,10 +51,10 @@ class SheetSynchronizer {
* credential email address.
*
* <p>The algorithm works by first assuming that the spreadsheet is sorted in the same way that
* {@code data} is sorted. It then iterates through the existing rows and comparing them to the
* items in {@code data}. Iteration continues until we either run out of rows, or items in
* {@code data}. If there's any rows remaining, they'll be deleted. If instead, items remain in
* data, they'll be inserted.
* {@code data} is sorted (i.e. registrar name order). It then iterates through the existing rows
* and comparing them to the items in {@code data}. Iteration continues until we either run out of
* rows, or items in {@code data}. If there's any rows remaining, they'll be deleted. If instead,
* items remain in data, they'll be appended to the end of the sheet.
*
* @param spreadsheetId The ID of your spreadsheet. This can be obtained by opening the Google
* spreadsheet in your browser and copying the ID from the URL.
@@ -61,62 +62,127 @@ class SheetSynchronizer {
* spreadsheet. Each row is a map, where the key must be exactly the same as the column header
* cell in the spreadsheet, and value is an arbitrary object which will be converted to a
* string before storing it in the spreadsheet.
* @throws IOException error communicating with the GData service.
* @throws ServiceException if a system error occurred when retrieving the entry.
* @throws com.google.gdata.util.ParseException error parsing the returned entry.
* @throws com.google.gdata.util.ResourceNotFoundException if an entry URL is not valid.
* @throws com.google.gdata.util.ServiceForbiddenException if the GData service cannot get the
* entry resource due to access constraints.
* @see <a href="https://developers.google.com/google-apps/spreadsheets/">Google Sheets API</a>
* @throws IOException if encountering an error communicating with the Sheets service.
* @see <a href="https://developers.google.com/sheets/">Google Sheets API v4</a>
*/
void synchronize(String spreadsheetId, ImmutableList<ImmutableMap<String, String>> data)
throws IOException, ServiceException {
URL url = new URL(SPREADSHEET_URL_PREFIX + spreadsheetId);
SpreadsheetEntry spreadsheet = spreadsheetService.getEntry(url, SpreadsheetEntry.class);
WorksheetEntry worksheet = spreadsheet.getWorksheets().get(0);
worksheet.setRowCount(data.size() + 1); // account for header row
worksheet = worksheet.update();
final ListFeed listFeed =
spreadsheetService.getFeed(worksheet.getListFeedUrl(), ListFeed.class);
List<ListEntry> entries = listFeed.getEntries();
int commonSize = Math.min(entries.size(), data.size());
for (int i = 0; i < commonSize; i++) {
final ListEntry entry = entries.get(i);
CustomElementCollection elements = entry.getCustomElements();
throws IOException {
// Get the existing sheet's values
ValueRange sheetValues =
sheetsService.spreadsheets().values().get(spreadsheetId, SHEET_NAME).execute();
List<List<Object>> originalVals = sheetValues.getValues();
// Assemble headers from the sheet
ImmutableList.Builder<String> headersBuilder = new ImmutableList.Builder<>();
for (Object headerCell : originalVals.get(0)) {
headersBuilder.add(headerCell.toString());
}
ImmutableList<String> headers = headersBuilder.build();
// Pop off the headers row
originalVals.remove(0);
List<ValueRange> updates = new ArrayList<>();
int minSize = Math.min(originalVals.size(), data.size());
for (int i = 0; i < minSize; i++) {
boolean mutated = false;
for (ImmutableMap.Entry<String, String> cell : data.get(i).entrySet()) {
if (!cell.getValue().equals(elements.getValue(cell.getKey()))) {
List<Object> cellRow = originalVals.get(i);
// If the row isn't full, pad it with empty strings until it is
while (cellRow.size() < headers.size()) {
cellRow.add("");
}
for (int j = 0; j < headers.size(); j++) {
// Look for the value corresponding to the row and header indices in data
String dataField = data.get(i).get(headers.get(j));
// If the cell's header matches a data header, and the values aren't equal, mutate it
if (dataField != null && !cellRow.get(j).toString().equals(dataField)) {
mutated = true;
elements.setValueLocal(cell.getKey(), cell.getValue());
originalVals.get(i).set(j, dataField);
}
}
if (mutated) {
// Wrap in a retrier to deal with transient HTTP failures, which are IOExceptions wrapped
// in RuntimeExceptions.
retrier.callWithRetry(new Callable<Void>() {
@Override
public Void call() throws Exception {
entry.update();
return null;
}}, RuntimeException.class);
ValueRange rowUpdate =
new ValueRange()
.setValues(originalVals.subList(i, i + 1))
.setRange(getCellRange(i));
updates.add(rowUpdate);
}
}
if (data.size() > entries.size()) {
for (int i = entries.size(); i < data.size(); i++) {
final ListEntry entry = listFeed.createEntry();
CustomElementCollection elements = entry.getCustomElements();
for (ImmutableMap.Entry<String, String> cell : data.get(i).entrySet()) {
elements.setValueLocal(cell.getKey(), cell.getValue());
// Update the mutated cells if necessary
if (!updates.isEmpty()) {
BatchUpdateValuesRequest updateRequest = new BatchUpdateValuesRequest()
.setValueInputOption("RAW")
.setData(updates);
BatchUpdateValuesResponse response =
sheetsService.spreadsheets().values().batchUpdate(spreadsheetId, updateRequest).execute();
Integer cellsUpdated = response.getTotalUpdatedCells();
logger.infofmt("Updated %d originalVals", cellsUpdated != null ? cellsUpdated : 0);
}
// Append extra rows if necessary
if (data.size() > originalVals.size()) {
ImmutableList.Builder<List<Object>> valsBuilder = new ImmutableList.Builder<>();
for (int i = originalVals.size(); i < data.size(); i++) {
ImmutableList.Builder<Object> rowBuilder = new ImmutableList.Builder<>();
for (String header : headers) {
rowBuilder.add(nullToEmpty(data.get(i).get(header)));
}
// Wrap in a retrier to deal with transient HTTP failures, which are IOExceptions wrapped
// in RuntimeExceptions.
retrier.callWithRetry(new Callable<Void>() {
@Override
public Void call() throws Exception {
listFeed.insert(entry);
return null;
}}, RuntimeException.class);
valsBuilder.add(rowBuilder.build());
}
// Start the append at index originalVals.size (where the previous operation left off)
ValueRange appendUpdate = new ValueRange().setValues(valsBuilder.build());
AppendValuesResponse appendResponse = sheetsService
.spreadsheets()
.values()
.append(spreadsheetId, getCellRange(originalVals.size()), appendUpdate)
.setValueInputOption("RAW")
.setInsertDataOption("INSERT_ROWS")
.execute();
logger.infofmt(
"Appended %d rows to range %s",
data.size() - originalVals.size(), appendResponse.getTableRange());
// Clear the extra rows if necessary
} else if (data.size() < originalVals.size()) {
// Clear other rows if there's more originalVals on the sheet than live data.
ClearValuesResponse clearResponse =
sheetsService
.spreadsheets()
.values()
.clear(
spreadsheetId,
getRowRange(data.size(), originalVals.size()),
new ClearValuesRequest())
.execute();
logger.infofmt(
"Cleared %d rows from range %s",
originalVals.size() - data.size(), clearResponse.getClearedRange());
}
}
/**
* Returns an A1 representation of the cell indicating the top-left corner of the update.
*
* <p>Updates and appends can specify either a complete range (i.e. A1:B4), or just the top-left
* corner of the request. For the latter case, the data fills in based on the primary dimension
* (either row-major or column-major order, default is row-major.) For simplicity, we just specify
* a single cell for these requests.
*
* @see <a href="https://developers.google.com/sheets/api/guides/values#writing_to_a_single_range">
* Writing to a single range</a>
*/
private String getCellRange(int rowNum) {
// We add 1 to rowNum to compensate for Sheet's 1-indexing, and 1 to offset for the header
return String.format("%s!A%d", SHEET_NAME, rowNum + 2);
}
/**
* Returns an A1 representation of the cell indicating an entire set of rows.
*
* <p>Clear requests require a specific range, and we always want to clear an entire row at a time
* (to avoid leaving any data behind).
*/
private String getRowRange(int firstRow, int lastRow) {
return String.format("%s!%d:%d", SHEET_NAME, firstRow + 2, lastRow + 2);
}
}

26
java/google/registry/export/sheet/SpreadsheetServiceModule.java → java/google/registry/export/sheet/SheetsServiceModule.java
View File

@@ -15,24 +15,28 @@
package google.registry.export.sheet;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.services.sheets.v4.Sheets;
import com.google.common.collect.ImmutableList;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
import dagger.Module;
import dagger.Provides;
import google.registry.config.RegistryConfig.Config;
/** Dagger module for {@link SpreadsheetService}. */
/** Dagger module for {@link Sheets}. */
@Module
public final class SpreadsheetServiceModule {
public final class SheetsServiceModule {
private static final String APPLICATION_NAME = "google-registry-v1";
private static final ImmutableList<String> SCOPES = ImmutableList.of(
"https://spreadsheets.google.com/feeds",
"https://docs.google.com/feeds");
"https://www.googleapis.com/auth/spreadsheets");
@Provides
static SpreadsheetService provideSpreadsheetService(GoogleCredential credential) {
SpreadsheetService service = new SpreadsheetService(APPLICATION_NAME);
service.setOAuth2Credentials(credential.createScoped(SCOPES));
return service;
static Sheets provideSheets(
HttpTransport transport,
JsonFactory jsonFactory,
@Config("projectId") String projectId,
GoogleCredential credential) {
return new Sheets.Builder(transport, jsonFactory, credential.createScoped(SCOPES))
.setApplicationName(projectId)
.build();
}
}

3
java/google/registry/export/sheet/SyncRegistrarsSheet.java
View File

@@ -34,7 +34,6 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Ordering;
import com.google.gdata.util.ServiceException;
import com.googlecode.objectify.VoidWork;
import google.registry.model.common.Cursor;
import google.registry.model.registrar.Registrar;
@@ -74,7 +73,7 @@ class SyncRegistrarsSheet {
}
/** Performs the synchronization operation. */
void run(String spreadsheetId) throws IOException, ServiceException {
void run(String spreadsheetId) throws IOException {
final DateTime executionTime = clock.nowUtc();
sheetSynchronizer.synchronize(
spreadsheetId,

4
java/google/registry/export/sheet/SyncRegistrarsSheetAction.java
View File

@@ -28,7 +28,6 @@ import com.google.appengine.api.modules.ModulesServiceFactory;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.base.Optional;
import com.google.gdata.util.ServiceException;
import google.registry.config.RegistryConfig.Config;
import google.registry.request.Action;
import google.registry.request.Parameter;
@@ -133,6 +132,7 @@ public class SyncRegistrarsSheetAction implements Runnable {
return;
}
}
String sheetLockName = String.format("%s: %s", LOCK_NAME, sheetId.get());
Callable<Void> runner = new Callable<Void>() {
@Nullable
@@ -141,7 +141,7 @@ public class SyncRegistrarsSheetAction implements Runnable {
try {
syncRegistrarsSheet.run(sheetId.get());
Result.OK.send(response, null);
} catch (IOException | ServiceException e) {
} catch (IOException e) {
Result.FAILED.send(response, e);
}
return null;