File tree

1 file changed

+34
-6
lines changed

1 file changed

+34
-6
lines changed
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import com.google.api.gax.longrunning.OperationFuture;
2323
import com.google.api.gax.paging.Page;
24+
import com.google.api.gax.rpc.FailedPreconditionException;
2425
import com.google.cloud.Timestamp;
2526
import com.google.cloud.spanner.Backup;
2627
import com.google.cloud.spanner.BackupId;
@@ -475,13 +476,40 @@ private void testRestore(Backup backup, OperationFuture<Backup, CreateBackupMeta
475476
throws InterruptedException, ExecutionException {
476477
// Restore the backup to a new database.
477478
String restoredDb = testHelper.getUniqueDatabaseId();
478-
logger.info(
479-
String.format(
480-
"Restoring backup %s to database %s", backup.getId().getBackup(), restoredDb));
481-
OperationFuture<Database, RestoreDatabaseMetadata> restoreOp =
482-
backup.restore(DatabaseId.of(testHelper.getInstanceId(), restoredDb));
479+
String restoreOperationName;
480+
OperationFuture<Database, RestoreDatabaseMetadata> restoreOp;
481+
int attempts = 0;
482+
while (true) {
483+
try {
484+
logger.info(
485+
String.format(
486+
"Restoring backup %s to database %s", backup.getId().getBackup(), restoredDb));
487+
restoreOp = backup.restore(DatabaseId.of(testHelper.getInstanceId(), restoredDb));
488+
restoreOperationName = restoreOp.getName();
489+
break;
490+
} catch (ExecutionException e) {
491+
if (e.getCause() instanceof FailedPreconditionException
492+
&& e.getCause()
493+
.getMessage()
494+
.contains("Please retry the operation once the pending restores complete")) {
495+
attempts++;
496+
if (attempts == 10) {
497+
logger.info(
498+
"Restore operation failed 10 times because of other pending restores. Skipping restore test.");
499+
return;
500+
}
501+
// wait and then retry.
502+
logger.info(
503+
String.format(
504+
"Restoring backup %s to database %s must wait because of other pending restore operation",
505+
backup.getId().getBackup(), restoredDb));
506+
Thread.sleep(60_000L);
507+
} else {
508+
throw e;
509+
}
510+
}
511+
}
483512
databases.add(restoredDb);
484-
final String restoreOperationName = restoreOp.getName();
485513
logger.info(String.format("Restore operation %s running", restoreOperationName));
486514
RestoreDatabaseMetadata metadata = restoreOp.getMetadata().get();
487515
assertThat(metadata.getBackupInfo().getBackup()).isEqualTo(backup.getId().getName());

0 commit comments

Comments
 (0)