@Public @Stable public static AllocateRequest newInstance(int responseID, float appProgress, List<ResourceRequest> resourceAsk, List<ContainerId> containersToBeReleased, ResourceBlacklistRequest resourceBlacklistRequest, List<ContainerResourceIncreaseRequest> increaseRequests) { AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); allocateRequest.setResponseId(responseID); allocateRequest.setProgress(appProgress); allocateRequest.setAskList(resourceAsk); allocateRequest.setReleaseList(containersToBeReleased); allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest); allocateRequest.setIncreaseRequests(increaseRequests); return allocateRequest; }
@Public @Stable public static AllocateRequest newInstance(int responseID, float appProgress, List<ResourceRequest> resourceAsk, List<ContainerId> containersToBeReleased, ResourceBlacklistRequest resourceBlacklistRequest, List<ContainerResourceChangeRequest> increaseRequests, List<ContainerResourceChangeRequest> decreaseRequests) { AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); allocateRequest.setResponseId(responseID); allocateRequest.setProgress(appProgress); allocateRequest.setAskList(resourceAsk); allocateRequest.setReleaseList(containersToBeReleased); allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest); allocateRequest.setIncreaseRequests(increaseRequests); allocateRequest.setDecreaseRequests(decreaseRequests); return allocateRequest; }
@Override public ResourceBlacklistRequest getBlacklistUpdates() { ResourceBlacklistRequest ret; List<String> blacklist = new ArrayList<>(blacklistNodes); final int currentBlacklistSize = blacklist.size(); final double failureThreshold = this.blacklistDisableFailureThreshold * numberOfNodeManagerHosts; if (currentBlacklistSize < failureThreshold) { if (LOG.isDebugEnabled()) { LOG.debug("blacklist size " + currentBlacklistSize + " is less than " + "failure threshold ratio " + blacklistDisableFailureThreshold + " out of total usable nodes " + numberOfNodeManagerHosts); } ret = ResourceBlacklistRequest.newInstance(blacklist, EMPTY_LIST); } else { LOG.warn("Ignoring Blacklists, blacklist size " + currentBlacklistSize + " is more than failure threshold ratio " + blacklistDisableFailureThreshold + " out of total usable nodes " + numberOfNodeManagerHosts); // TODO: After the threshold hits, we will keep sending a long list // every time a new AM is to be scheduled. ret = ResourceBlacklistRequest.newInstance(EMPTY_LIST, blacklist); } return ret; }
@Test public void testSimpleBlacklistBelowFailureThreshold() { final int numberOfNodeManagerHosts = 3; final double blacklistDisableFailureThreshold = 0.8; BlacklistManager manager = new SimpleBlacklistManager( numberOfNodeManagerHosts, blacklistDisableFailureThreshold); String anyNode = "foo"; String anyNode2 = "bar"; manager.addNode(anyNode); manager.addNode(anyNode2); ResourceBlacklistRequest blacklist = manager .getBlacklistUpdates(); List<String> blacklistAdditions = blacklist.getBlacklistAdditions(); Collections.sort(blacklistAdditions); List<String> blacklistRemovals = blacklist.getBlacklistRemovals(); String[] expectedBlacklistAdditions = new String[]{anyNode2, anyNode}; Assert.assertArrayEquals( "Blacklist additions was not as expected", expectedBlacklistAdditions, blacklistAdditions.toArray()); Assert.assertTrue( "Blacklist removals should be empty but was " + blacklistRemovals, blacklistRemovals.isEmpty()); }
@Test public void testDisabledBlacklist() { BlacklistManager disabled = new DisabledBlacklistManager(); String anyNode = "foo"; disabled.addNode(anyNode); ResourceBlacklistRequest blacklist = disabled .getBlacklistUpdates(); List<String> blacklistAdditions = blacklist.getBlacklistAdditions(); List<String> blacklistRemovals = blacklist.getBlacklistRemovals(); Assert.assertTrue( "Blacklist additions should be empty but was " + blacklistAdditions, blacklistAdditions.isEmpty()); Assert.assertTrue( "Blacklist removals should be empty but was " + blacklistRemovals, blacklistRemovals.isEmpty()); }
@Test(timeout = 15000) public void testAllocateOnHA() throws YarnException, IOException, InterruptedException { AllocateRequest request = AllocateRequest.newInstance(0, 50f, new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>(), ResourceBlacklistRequest.newInstance(new ArrayList<String>(), new ArrayList<String>())); int nbTry = 0; AllocateResponse response = null; while (nbTry < 10) { try { response = amClient.allocate(request); break; } catch (IOException ex) { if (!(ex instanceof SecretManager.InvalidToken)) { throw ex; } } Thread.sleep(200); nbTry++; } Assert.assertEquals(response, this.cluster.createFakeAllocateResponse()); }
/** * Use {@link AllocateRequest#newInstance(int, float, List, List, * ResourceBlacklistRequest, List)} instead * @param responseID responseId * @param appProgress appProgress * @param resourceAsk resourceAsk * @param containersToBeReleased containersToBeReleased * @param resourceBlacklistRequest resourceBlacklistRequest * @param increaseRequests increaseRequests * @return AllocateRequest */ @Deprecated public static AllocateRequest newInstance(int responseID, float appProgress, List<ResourceRequest> resourceAsk, List<ContainerId> containersToBeReleased, ResourceBlacklistRequest resourceBlacklistRequest, List<ContainerResourceIncreaseRequest> increaseRequests) { AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); allocateRequest.setResponseId(responseID); allocateRequest.setProgress(appProgress); allocateRequest.setAskList(resourceAsk); allocateRequest.setReleaseList(containersToBeReleased); allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest); allocateRequest.setIncreaseRequests(increaseRequests); return allocateRequest; }
@Public @Unstable public static AllocateRequest newInstance(int responseID, float appProgress, List<ResourceRequest> resourceAsk, List<ContainerId> containersToBeReleased, List<UpdateContainerRequest> updateRequests, ResourceBlacklistRequest resourceBlacklistRequest) { AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); allocateRequest.setResponseId(responseID); allocateRequest.setProgress(appProgress); allocateRequest.setAskList(resourceAsk); allocateRequest.setReleaseList(containersToBeReleased); allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest); allocateRequest.setUpdateRequests(updateRequests); return allocateRequest; }
@Override public ResourceBlacklistRequest getResourceBlacklistRequest() { AllocateRequestProtoOrBuilder p = viaProto ? proto : builder; if (this.blacklistRequest != null) { return this.blacklistRequest; } if (!p.hasBlacklistRequest()) { return null; } this.blacklistRequest = convertFromProtoFormat(p.getBlacklistRequest()); return this.blacklistRequest; }
@Override public void setResourceBlacklistRequest(ResourceBlacklistRequest blacklistRequest) { maybeInitBuilder(); if (blacklistRequest == null) { builder.clearBlacklistRequest(); } this.blacklistRequest = blacklistRequest; }
public static void validateBlacklistRequest( ResourceBlacklistRequest blacklistRequest) throws InvalidResourceBlacklistRequestException { if (blacklistRequest != null) { List<String> plus = blacklistRequest.getBlacklistAdditions(); if (plus != null && plus.contains(ResourceRequest.ANY)) { throw new InvalidResourceBlacklistRequestException( "Cannot add " + ResourceRequest.ANY + " to the blacklist!"); } } }
@Test(timeout = 15000) public void testAllocateOnHA() throws YarnException, IOException { AllocateRequest request = AllocateRequest.newInstance(0, 50f, new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>(), ResourceBlacklistRequest.newInstance(new ArrayList<String>(), new ArrayList<String>())); AllocateResponse response = amClient.allocate(request); Assert.assertEquals(response, this.cluster.createFakeAllocateResponse()); }
@Public @Stable public static AllocateRequest newInstance(int responseID, float appProgress, List<ResourceRequest> resourceAsk, List<ContainerId> containersToBeReleased, ResourceBlacklistRequest resourceBlacklistRequest) { return newInstance(responseID, appProgress, resourceAsk, containersToBeReleased, resourceBlacklistRequest, null); }
@Public @Stable public static AllocateRequest newInstance(int responseID, float appProgress, List<ResourceRequest> resourceAsk, List<ContainerId> containersToBeReleased, ResourceBlacklistRequest resourceBlacklistRequest) { return newInstance(responseID, appProgress, resourceAsk, containersToBeReleased, resourceBlacklistRequest, null, null); }
protected AllocateResponse makeRemoteRequest() throws YarnException, IOException { ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions), new ArrayList<String>(blacklistRemovals)); AllocateRequest allocateRequest = AllocateRequest.newInstance(lastResponseID, super.getApplicationProgress(), new ArrayList<ResourceRequest>(ask), new ArrayList<ContainerId>(release), blacklistRequest); AllocateResponse allocateResponse = scheduler.allocate(allocateRequest); lastResponseID = allocateResponse.getResponseId(); availableResources = allocateResponse.getAvailableResources(); lastClusterNmCount = clusterNmCount; clusterNmCount = allocateResponse.getNumClusterNodes(); if (ask.size() > 0 || release.size() > 0) { LOG.info("getResources() for " + applicationId + ":" + " ask=" + ask.size() + " release= " + release.size() + " newContainers=" + allocateResponse.getAllocatedContainers().size() + " finishedContainers=" + allocateResponse.getCompletedContainersStatuses().size() + " resourcelimit=" + availableResources + " knownNMs=" + clusterNmCount); } ask.clear(); release.clear(); if (blacklistAdditions.size() > 0 || blacklistRemovals.size() > 0) { LOG.info("Update the blacklist for " + applicationId + ": blacklistAdditions=" + blacklistAdditions.size() + " blacklistRemovals=" + blacklistRemovals.size()); } blacklistAdditions.clear(); blacklistRemovals.clear(); return allocateResponse; }
public static void validateBlacklistRequest(ResourceBlacklistRequest blacklistRequest) throws InvalidResourceBlacklistRequestException { if (blacklistRequest != null) { List<String> plus = blacklistRequest.getBlacklistAdditions(); if (plus != null && plus.contains(ResourceRequest.ANY)) { throw new InvalidResourceBlacklistRequestException( "Cannot add " + ResourceRequest.ANY + " to the blacklist!"); } } }
@Public @Stable public static AllocateRequest newInstance(int responseID, float appProgress, List<ResourceRequest> resourceAsk, List<ContainerId> containersToBeReleased, ResourceBlacklistRequest resourceBlacklistRequest) { AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); allocateRequest.setResponseId(responseID); allocateRequest.setProgress(appProgress); allocateRequest.setAskList(resourceAsk); allocateRequest.setReleaseList(containersToBeReleased); allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest); return allocateRequest; }
private AllocateRequest createAllocateRequest(List<NodeReport> listNode) { // The test needs AMRMClient to create a real allocate request AMRMClientImpl<ContainerRequest> amClient = new AMRMClientImpl<ContainerRequest>(); Resource capability = Resource.newInstance(1024, 2); Priority priority = Priority.newInstance(1); List<NodeReport> nodeReports = listNode; String node = nodeReports.get(0).getNodeId().getHost(); String[] nodes = new String[] { node }; ContainerRequest storedContainer1 = new ContainerRequest(capability, nodes, null, priority); amClient.addContainerRequest(storedContainer1); amClient.addContainerRequest(storedContainer1); List<ResourceRequest> resourceAsk = new ArrayList<ResourceRequest>(); for (ResourceRequest rr : amClient.ask) { resourceAsk.add(rr); } ResourceBlacklistRequest resourceBlacklistRequest = ResourceBlacklistRequest .newInstance(new ArrayList<String>(), new ArrayList<String>()); int responseId = 1; return AllocateRequest.newInstance(responseId, 0, resourceAsk, new ArrayList<ContainerId>(), resourceBlacklistRequest); }