Merged

fix: retry restore if blocked by pending restore #119

Changes from all commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Failed to load files.
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,7 @@

import com.google.api.gax.longrunning.OperationFuture;
import com.google.api.gax.paging.Page;
import com.google.api.gax.rpc.FailedPreconditionException;
import com.google.cloud.Timestamp;
import com.google.cloud.spanner.Backup;
import com.google.cloud.spanner.BackupId;
Expand DownExpand Up@@ -475,13 +476,40 @@ private void testRestore(Backup backup, OperationFuture<Backup, CreateBackupMeta
throws InterruptedException, ExecutionException {
// Restore the backup to a new database.
String restoredDb = testHelper.getUniqueDatabaseId();
logger.info(
String.format(
"Restoring backup %s to database %s", backup.getId().getBackup(), restoredDb));
OperationFuture<Database, RestoreDatabaseMetadata> restoreOp =
backup.restore(DatabaseId.of(testHelper.getInstanceId(), restoredDb));
String restoreOperationName;
OperationFuture<Database, RestoreDatabaseMetadata> restoreOp;
int attempts = 0;
while (true) {
try {
logger.info(
String.format(
"Restoring backup %s to database %s", backup.getId().getBackup(), restoredDb));
restoreOp = backup.restore(DatabaseId.of(testHelper.getInstanceId(), restoredDb));
restoreOperationName = restoreOp.getName();
break;
} catch (ExecutionException e) {
if (e.getCause() instanceof FailedPreconditionException
&& e.getCause()
.getMessage()
.contains("Please retry the operation once the pending restores complete")) {
attempts++;
if (attempts == 10) {
logger.info(
"Restore operation failed 10 times because of other pending restores. Skipping restore test.");
return;
}
// wait and then retry.
logger.info(
String.format(
"Restoring backup %s to database %s must wait because of other pending restore operation",
backup.getId().getBackup(), restoredDb));
Thread.sleep(60_000L);
} else {
throw e;
}
}
}
databases.add(restoredDb);
final String restoreOperationName = restoreOp.getName();
logger.info(String.format("Restore operation %s running", restoreOperationName));
RestoreDatabaseMetadata metadata = restoreOp.getMetadata().get();
assertThat(metadata.getBackupInfo().getBackup()).isEqualTo(backup.getId().getName());
Expand Down