mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-09-12 11:06:29 +00:00
Code Issues Releases Wiki Activity

Delete code relating to SQL init and scheduling (#1661)

Browse Source 
One of the more significant changes introduced in this PR is that we use
SQL as the backing database in all tests unless otherwise specified,
e.g. by using the TmOverrideExtension. We change various ofy-related
tests to use this.

This includes various changes:
- Deletion of SqlEntity/DatastoreEntity and related classes. Includes
  any necessary changes because of that (e.g. getting a nice SQL key on
  error in RegistryJpaIO).
- Deletion of classes that used libraries from the init-sql code
  (RefreshDnsOnHostRenameAction)
- Removal of the JpaTransactionManager's backup implementation
- Modification of RegistryJpaWriteTest to not use init-sql code
- Removal of the Transaction class and related classes, however it does
  not remove the TransactionEntity class as that would require DB
  changes
- Removal of anything related to the actual usage of the database
  migration schedule or read-only phases
- Various test changes and fixes to account for the differences in SQL
  (like how foreign keys need to exist)

This deliberately doesn't do anything to alter the objects actually
stored in the DB yet, just how we use them
This commit is contained in:
gbrodman
2022-06-13 15:10:35 -04:00
committed by GitHub
parent dcc11379c8
commit 2f8be045c7
232 changed files with 457 additions and 8245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/resources/google/registry/beam/init_sql_pipeline_metadata.json
-68
View File
@@ -1,68 +0,0 @@
{
"name": "Init SQL From Datastore Backup",
"description": "An Apache Beam batch pipeline that reads a Datastore export and Nomulus CommitLog files, transforms data into JPA entities, and writes the results to a SQL database.",
"parameters": [
{
"name": "registryEnvironment",
"label": "The Registry environment.",
"helpText": "The Registry environment.",
"is_optional": false,
"regexes": [
"^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"
]
},
{
"name": "isolationOverride",
"label": "The desired SQL transaction isolation level.",
"helpText": "The desired SQL transaction isolation level.",
"is_optional": true,
"regexes": [
"^[0-9A-Z_]+$"
]
},
{
"name": "sqlWriteBatchSize",
"label": "SQL write batch size.",
"helpText": "The number of entities to write to the SQL database in one operation.",
"is_optional": true,
"regexes": [
"^[1-9][0-9]*$"
]
},
{
"name": "sqlWriteShards",
"label": "Number of output shards to create when writing to SQL.",
"helpText": "Number of shards to create out of the data before writing to the SQL database. Please refer to the Javadoc of RegistryJpaIO.Write.shards() for how to choose this value.",
"is_optional": true,
"regexes": [
"^[1-9][0-9]*$"
]
},
{
"name": "datastoreExportDir",
"label": "Datastore export dir",
"helpText": "The root directory of the export to load.",
"regexes": [
"^gs:\\/\\/[^\\n\\r]+$"
]
},
{
"name": "commitLogDir",
"label": "Nomulus CommitLog dir",
"helpText": "The directory with all Nomulus CommitLogs.",
"regexes": [
"^gs:\\/\\/[^\\n\\r]+$"
]
},
{
"name": "commitLogStartTimestamp",
"label": "Nomulus CommitLog start time",
"helpText": "The earliest CommitLogs to load, in ISO8601 format."
},
{
"name": "commitLogEndTimestamp",
"label": "Nomulus CommitLog end time",
"helpText": "The latest CommitLogs to load, in ISO8601 format."
}
]
}
core/src/main/resources/google/registry/beam/invoicing_pipeline_metadata.json
-8
View File
@@ -62,14 +62,6 @@
"^gs:\\/\\/[^\\n\\r]+$"
]
},
{
"name": "database",
"label": "Database to read from.",
"helpText": "DATASTORE or CLOUD_SQL.",
"regexes": [
"^DATASTORE|CLOUD_SQL$"
]
},
{
"name": "jpaTransactionManagerType",
"label": "The type of JPA transaction manager to use if using SQL",
core/src/main/resources/google/registry/beam/spec11_pipeline_metadata.json
-8
View File
@@ -61,14 +61,6 @@
"regexes": [
"^gs:\\/\\/[^\\n\\r]+$"
]
},
{
"name": "database",
"label": "Database to read from.",
"helpText": "DATASTORE or CLOUD_SQL.",
"regexes": [
"^DATASTORE|CLOUD_SQL$"
]
}
]
}
core/src/main/resources/google/registry/beam/validate_database_pipeline_metadata.json
-48
View File
@@ -1,48 +0,0 @@
{
"name": "Validate Datastore and Cloud SQL",
"description": "An Apache Beam batch pipeline that compares the data in Datastore and Cloud SQL database.",
"parameters": [
{
"name": "registryEnvironment",
"label": "The Registry environment.",
"helpText": "The Registry environment.",
"is_optional": false,
"regexes": [
"^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"
]
},
{
"name": "isolationOverride",
"label": "The desired SQL transaction isolation level.",
"helpText": "The desired SQL transaction isolation level.",
"is_optional": true,
"regexes": [
"^[0-9A-Z_]+$"
]
},
{
"name": "sqlSnapshotId",
"label": "The ID of an exported Cloud SQL (Postgresql) snapshot.",
"helpText": "The ID of an exported Cloud SQL (Postgresql) snapshot.",
"is_optional": false
},
{
"name": "latestCommitLogTimestamp",
"label": "Nomulus CommitLog start time",
"helpText": "The latest entity update time allowed for inclusion in validation, in ISO8601 format.",
"is_optional": false
},
{
"name": "comparisonStartTimestamp",
"label": "Only entities updated at or after this time are included for validation.",
"helpText": "The earliest entity update time allowed for inclusion in validation, in ISO8601 format.",
"is_optional": true
},
{
"name": "diffOutputGcsBucket",
"label": "The GCS bucket where data discrepancies should be output to.",
"helpText": "The GCS bucket where data discrepancies should be output to.",
"is_optional": true
}
]
}