|
21 | 21 |
|
22 | 22 | import com.google.api.gax.longrunning.OperationFuture;
|
23 | 23 | import com.google.api.gax.paging.Page;
|
| 24 | +import com.google.api.gax.rpc.FailedPreconditionException; |
24 | 25 | import com.google.cloud.Timestamp;
|
25 | 26 | import com.google.cloud.spanner.Backup;
|
26 | 27 | import com.google.cloud.spanner.BackupId;
|
@@ -475,13 +476,40 @@ private void testRestore(Backup backup, OperationFuture<Backup, CreateBackupMeta
|
475 | 476 | throws InterruptedException, ExecutionException {
|
476 | 477 | // Restore the backup to a new database.
|
477 | 478 | String restoredDb = testHelper.getUniqueDatabaseId();
|
478 |
| -logger.info( |
479 |
| -String.format( |
480 |
| -"Restoring backup %s to database %s", backup.getId().getBackup(), restoredDb)); |
481 |
| -OperationFuture<Database, RestoreDatabaseMetadata> restoreOp = |
482 |
| -backup.restore(DatabaseId.of(testHelper.getInstanceId(), restoredDb)); |
| 479 | +String restoreOperationName; |
| 480 | +OperationFuture<Database, RestoreDatabaseMetadata> restoreOp; |
| 481 | +int attempts = 0; |
| 482 | +while (true) { |
| 483 | +try { |
| 484 | +logger.info( |
| 485 | +String.format( |
| 486 | +"Restoring backup %s to database %s", backup.getId().getBackup(), restoredDb)); |
| 487 | +restoreOp = backup.restore(DatabaseId.of(testHelper.getInstanceId(), restoredDb)); |
| 488 | +restoreOperationName = restoreOp.getName(); |
| 489 | +break; |
| 490 | +} catch (ExecutionException e) { |
| 491 | +if (e.getCause() instanceof FailedPreconditionException |
| 492 | +&& e.getCause() |
| 493 | +.getMessage() |
| 494 | +.contains("Please retry the operation once the pending restores complete")) { |
| 495 | +attempts++; |
| 496 | +if (attempts == 10) { |
| 497 | +logger.info( |
| 498 | +"Restore operation failed 10 times because of other pending restores. Skipping restore test."); |
| 499 | +return; |
| 500 | +} |
| 501 | +// wait and then retry. |
| 502 | +logger.info( |
| 503 | +String.format( |
| 504 | +"Restoring backup %s to database %s must wait because of other pending restore operation", |
| 505 | +backup.getId().getBackup(), restoredDb)); |
| 506 | +Thread.sleep(60_000L); |
| 507 | +} else { |
| 508 | +throw e; |
| 509 | +} |
| 510 | +} |
| 511 | +} |
483 | 512 | databases.add(restoredDb);
|
484 |
| -final String restoreOperationName = restoreOp.getName(); |
485 | 513 | logger.info(String.format("Restore operation %s running", restoreOperationName));
|
486 | 514 | RestoreDatabaseMetadata metadata = restoreOp.getMetadata().get();
|
487 | 515 | assertThat(metadata.getBackupInfo().getBackup()).isEqualTo(backup.getId().getName());
|
|
0 commit comments