feat: retry admin request limit exceeded error

Automatically retry requests that fail because the admin requests per seconds
limit has been exceeded using an exponential backoff.

Fixes #655 and others

@@ -0,0 +1,33 @@
+/*
+* Copyright 2020 Google LLC
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package com.google.cloud.spanner;
+
+import javax.annotation.Nullable;
+
+/**
+* Exception thrown by Cloud Spanner the number of administrative requests per minute has been
+* exceeded.
+*/
+public class AdminRequestsPerMinuteExceededException extends SpannerException {
+private static final long serialVersionUID = -6395746612598975751L;
+
+/** Private constructor. Use {@link SpannerExceptionFactory} to create instances. */
+AdminRequestsPerMinuteExceededException(
+DoNotConstructDirectly token, @Nullable String message, @Nullable Throwable cause) {
+super(token, ErrorCode.RESOURCE_EXHAUSTED, true, message, cause);
+}
+}

@@ -22,6 +22,7 @@
 import com.google.cloud.spanner.SpannerException.DoNotConstructDirectly;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Predicate;
+import com.google.rpc.ErrorInfo;
 import com.google.rpc.ResourceInfo;
 import io.grpc.Context;
 import io.grpc.Metadata;
@@ -46,6 +47,8 @@ public final class SpannerExceptionFactory {
 "type.googleapis.com/google.spanner.admin.instance.v1.Instance";
 private static final Metadata.Key<ResourceInfo> KEY_RESOURCE_INFO =
 ProtoUtils.keyForProto(ResourceInfo.getDefaultInstance());
+private static final Metadata.Key<ErrorInfo> KEY_ERROR_INFO =
+ProtoUtils.keyForProto(ErrorInfo.getDefaultInstance());
 
 public static SpannerException newSpannerException(ErrorCode code, @Nullable String message) {
 return newSpannerException(code, message, null);
@@ -213,13 +216,31 @@ private static ResourceInfo extractResourceInfo(Throwable cause) {
 return null;
 }
 
+private static ErrorInfo extractErrorInfo(Throwable cause) {
+if (cause != null) {
+Metadata trailers = Status.trailersFromThrowable(cause);
+if (trailers != null) {
+return trailers.get(KEY_ERROR_INFO);
+}
+}
+return null;
+}
+
 static SpannerException newSpannerExceptionPreformatted(
 ErrorCode code, @Nullable String message, @Nullable Throwable cause) {
 // This is the one place in the codebase that is allowed to call constructors directly.
 DoNotConstructDirectly token = DoNotConstructDirectly.ALLOWED;
 switch (code) {
 case ABORTED:
 return new AbortedException(token, message, cause);
+case RESOURCE_EXHAUSTED:
+ErrorInfo info = extractErrorInfo(cause);
+if (info != null
+&& info.getMetadataMap().containsKey("quota_limit")
+&& "AdminMethodQuotaPerMinutePerProject"
+.equals(info.getMetadataMap().get("quota_limit"))) {
+return new AdminRequestsPerMinuteExceededException(token, message, cause);
+}
 case NOT_FOUND:
 ResourceInfo resourceInfo = extractResourceInfo(cause);
 if (resourceInfo != null) {