xds:Fix ConcurrentModificationException in PriorityLoadBalancer (#9728)

Fix ConcurrentModificationException in PriorityLoadBalancer by making copy of children values to iterate rather than directly using children in for loop.

@@ -37,6 +37,8 @@
 import io.grpc.xds.PriorityLoadBalancerProvider.PriorityLbConfig.PriorityChildConfig;
 import io.grpc.xds.XdsLogger.XdsLogLevel;
 import io.grpc.xds.XdsSubchannelPickers.ErrorPicker;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -59,6 +61,8 @@ final class PriorityLoadBalancer extends LoadBalancer {
 
 // Includes all active and deactivated children. Mutable. New entries are only added from priority
 // 0 up to the selected priority. An entry is only deleted 15 minutes after its deactivation.
+// Note that because all configuration updates should be atomic, updates to children can happen
+// outside of the synchronization context. Therefore copy values before looping over them.
 private final Map<String, ChildLbState> children = new HashMap<>();
 
 // Following fields are only null initially.
@@ -91,15 +95,20 @@ public boolean acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) {
 priorityNames = config.priorities;
 priorityConfigs = config.childConfigs;
 Set<String> prioritySet = new HashSet<>(config.priorities);
-for (String priority : children.keySet()) {
+ArrayList<String> childKeys = new ArrayList<>(children.keySet());
+for (String priority : childKeys) {
 if (!prioritySet.contains(priority)) {
-children.get(priority).deactivate();
+ChildLbState childLbState = children.get(priority);
+if (childLbState != null) {
+childLbState.deactivate();
+}
 }
 }
 handlingResolvedAddresses = true;
 for (String priority : priorityNames) {
-if (children.containsKey(priority)) {
-children.get(priority).updateResolvedAddresses();
+ChildLbState childLbState = children.get(priority);
+if (childLbState != null) {
+childLbState.updateResolvedAddresses();
 }
 }
 handlingResolvedAddresses = false;
@@ -111,7 +120,8 @@ public boolean acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) {
 public void handleNameResolutionError(Status error) {
 logger.log(XdsLogLevel.WARNING, "Received name resolution error: {0}", error);
 boolean gotoTransientFailure = true;
-for (ChildLbState child : children.values()) {
+Collection<ChildLbState> childValues = new ArrayList<>(children.values());
+for (ChildLbState child : childValues) {
 if (priorityNames.contains(child.priority)) {
 child.lb.handleNameResolutionError(error);
 gotoTransientFailure = false;
@@ -125,7 +135,8 @@ public void handleNameResolutionError(Status error) {
 @Override
 public void shutdown() {
 logger.log(XdsLogLevel.INFO, "Shutdown");
-for (ChildLbState child : children.values()) {
+Collection<ChildLbState> childValues = new ArrayList<>(children.values());
+for (ChildLbState child : childValues) {
 child.tearDown();
 }
 children.clear();