public NMToken createAndGetNMToken(String applicationSubmitter, ApplicationAttemptId appAttemptId, Container container) { try { this.readLock.lock(); HashSet<NodeId> nodeSet = this.appAttemptToNodeKeyMap.get(appAttemptId); NMToken nmToken = null; if (nodeSet != null) { if (!nodeSet.contains(container.getNodeId())) { LOG.info("Sending NMToken for nodeId : " + container.getNodeId() + " for container : " + container.getId()); Token token = createNMToken(container.getId().getApplicationAttemptId(), container.getNodeId(), applicationSubmitter); nmToken = NMToken.newInstance(container.getNodeId(), token); nodeSet.add(container.getNodeId()); } } return nmToken; } finally { this.readLock.unlock(); } }
@Public @Stable public static AllocateResponse newInstance(int responseId, List<ContainerStatus> completedContainers, List<Container> allocatedContainers, List<NodeReport> updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt, List<NMToken> nmTokens) { AllocateResponse response = Records.newRecord(AllocateResponse.class); response.setNumClusterNodes(numClusterNodes); response.setResponseId(responseId); response.setCompletedContainersStatuses(completedContainers); response.setAllocatedContainers(allocatedContainers); response.setUpdatedNodes(updatedNodes); response.setAvailableResources(availResources); response.setAMCommand(command); response.setPreemptionMessage(preempt); response.setNMTokens(nmTokens); return response; }
@Public @Stable public static AllocateResponse newInstance(int responseId, List<ContainerStatus> completedContainers, List<Container> allocatedContainers, List<NodeReport> updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt, List<NMToken> nmTokens, List<ContainerResourceIncrease> increasedContainers, List<ContainerResourceDecrease> decreasedContainers) { AllocateResponse response = newInstance(responseId, completedContainers, allocatedContainers, updatedNodes, availResources, command, numClusterNodes, preempt, nmTokens); response.setIncreasedContainers(increasedContainers); response.setDecreasedContainers(decreasedContainers); return response; }
@Private @Unstable public static AllocateResponse newInstance(int responseId, List<ContainerStatus> completedContainers, List<Container> allocatedContainers, List<NodeReport> updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken, List<ContainerResourceIncrease> increasedContainers, List<ContainerResourceDecrease> decreasedContainers) { AllocateResponse response = newInstance(responseId, completedContainers, allocatedContainers, updatedNodes, availResources, command, numClusterNodes, preempt, nmTokens, increasedContainers, decreasedContainers); response.setAMRMToken(amRMToken); return response; }
@Private @Unstable public static RegisterApplicationMasterResponse newInstance( Resource minCapability, Resource maxCapability, Map<ApplicationAccessType, String> acls, ByteBuffer key, List<Container> containersFromPreviousAttempt, String queue, List<NMToken> nmTokensFromPreviousAttempts) { RegisterApplicationMasterResponse response = Records.newRecord(RegisterApplicationMasterResponse.class); response.setMaximumResourceCapability(maxCapability); response.setApplicationACLs(acls); response.setClientToAMTokenMasterKey(key); response.setContainersFromPreviousAttempts(containersFromPreviousAttempt); response.setNMTokensFromPreviousAttempts(nmTokensFromPreviousAttempts); response.setQueue(queue); return response; }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { Assert.assertEquals("response ID mismatch", responseId, request.getResponseId()); ++responseId; org.apache.hadoop.yarn.api.records.Token yarnToken = null; if (amToken != null) { yarnToken = org.apache.hadoop.yarn.api.records.Token.newInstance( amToken.getIdentifier(), amToken.getKind().toString(), amToken.getPassword(), amToken.getService().toString()); } return AllocateResponse.newInstance(responseId, Collections.<ContainerStatus>emptyList(), Collections.<Container>emptyList(), Collections.<NodeReport>emptyList(), Resources.none(), null, 1, null, Collections.<NMToken>emptyList(), yarnToken, Collections.<ContainerResourceIncrease>emptyList(), Collections.<ContainerResourceDecrease>emptyList()); }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { lastAsk = request.getAskList(); for (ResourceRequest req : lastAsk) { if (ResourceRequest.ANY.equals(req.getResourceName())) { Priority priority = req.getPriority(); if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) { lastAnyAskMap = req.getNumContainers(); } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){ lastAnyAskReduce = req.getNumContainers(); } } } AllocateResponse response = AllocateResponse.newInstance( request.getResponseId(), containersToComplete, containersToAllocate, Collections.<NodeReport>emptyList(), Resource.newInstance(512000, 1024, 1024), null, 10, null, Collections.<NMToken>emptyList()); containersToComplete.clear(); containersToAllocate.clear(); return response; }
@Public @Stable public static AllocateResponse newInstance(int responseId, List<ContainerStatus> completedContainers, List<Container> allocatedContainers, List<NodeReport> updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt, List<NMToken> nmTokens, List<Container> increasedContainers, List<Container> decreasedContainers) { AllocateResponse response = newInstance(responseId, completedContainers, allocatedContainers, updatedNodes, availResources, command, numClusterNodes, preempt, nmTokens); response.setIncreasedContainers(increasedContainers); response.setDecreasedContainers(decreasedContainers); return response; }
@Private @Unstable public static AllocateResponse newInstance(int responseId, List<ContainerStatus> completedContainers, List<Container> allocatedContainers, List<NodeReport> updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken, List<Container> increasedContainers, List<Container> decreasedContainers) { AllocateResponse response = newInstance(responseId, completedContainers, allocatedContainers, updatedNodes, availResources, command, numClusterNodes, preempt, nmTokens, increasedContainers, decreasedContainers); response.setAMRMToken(amRMToken); return response; }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { Assert.assertEquals("response ID mismatch", responseId, request.getResponseId()); ++responseId; org.apache.hadoop.yarn.api.records.Token yarnToken = null; if (amToken != null) { yarnToken = org.apache.hadoop.yarn.api.records.Token.newInstance( amToken.getIdentifier(), amToken.getKind().toString(), amToken.getPassword(), amToken.getService().toString()); } AllocateResponse response = AllocateResponse.newInstance(responseId, Collections.<ContainerStatus>emptyList(), Collections.<Container>emptyList(), Collections.<NodeReport>emptyList(), Resources.none(), null, 1, null, Collections.<NMToken>emptyList(), yarnToken, Collections.<Container>emptyList(), Collections.<Container>emptyList()); response.setApplicationPriority(Priority.newInstance(0)); return response; }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { lastAsk = request.getAskList(); for (ResourceRequest req : lastAsk) { if (ResourceRequest.ANY.equals(req.getResourceName())) { Priority priority = req.getPriority(); if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) { lastAnyAskMap = req.getNumContainers(); } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){ lastAnyAskReduce = req.getNumContainers(); } } } AllocateResponse response = AllocateResponse.newInstance( request.getResponseId(), containersToComplete, containersToAllocate, Collections.<NodeReport>emptyList(), Resource.newInstance(512000, 1024), null, 10, null, Collections.<NMToken>emptyList()); // RM will always ensure that a default priority is sent to AM response.setApplicationPriority(Priority.newInstance(0)); containersToComplete.clear(); containersToAllocate.clear(); return response; }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { lastAsk = request.getAskList(); for (ResourceRequest req : lastAsk) { if (ResourceRequest.ANY.equals(req.getResourceName())) { Priority priority = req.getPriority(); if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) { lastAnyAskMap = req.getNumContainers(); } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){ lastAnyAskReduce = req.getNumContainers(); } } } AllocateResponse response = AllocateResponse.newInstance( request.getResponseId(), containersToComplete, containersToAllocate, Collections.<NodeReport>emptyList(), Resource.newInstance(512000, 1024), null, 10, null, Collections.<NMToken>emptyList()); containersToComplete.clear(); containersToAllocate.clear(); return response; }
protected void allocateContainersAndValidateNMTokens(MockAM am, ArrayList<Container> containersReceived, int totalContainerRequested, HashMap<String, Token> nmTokens) throws Exception, InterruptedException { ArrayList<ContainerId> releaseContainerList = new ArrayList<ContainerId>(); AllocateResponse response; ArrayList<ResourceRequest> resourceRequest = new ArrayList<ResourceRequest>(); while (containersReceived.size() < totalContainerRequested) { LOG.info("requesting containers.."); response = am.allocate(resourceRequest, releaseContainerList); containersReceived.addAll(response.getAllocatedContainers()); if (!response.getNMTokens().isEmpty()) { for (NMToken nmToken : response.getNMTokens()) { String nodeId = nmToken.getNodeId().toString(); if (nmTokens.containsKey(nodeId)) { Assert.fail("Duplicate NMToken received for : " + nodeId); } nmTokens.put(nodeId, nmToken.getToken()); } } LOG.info("Got " + containersReceived.size() + " containers. Waiting to get " + totalContainerRequested); Thread.sleep(500); } }
public NMToken createAndGetNMToken(String applicationSubmitter, ApplicationAttemptId appAttemptId, Container container) { try { this.writeLock.lock(); HashSet<NodeId> nodeSet = this.appAttemptToNodeKeyMap.get(appAttemptId); NMToken nmToken = null; if (nodeSet != null) { if (!nodeSet.contains(container.getNodeId())) { LOG.info("Sending NMToken for nodeId : " + container.getNodeId() + " for container : " + container.getId()); Token token = createNMToken(container.getId().getApplicationAttemptId(), container.getNodeId(), applicationSubmitter); nmToken = NMToken.newInstance(container.getNodeId(), token); nodeSet.add(container.getNodeId()); } } return nmToken; } finally { this.writeLock.unlock(); } }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { Assert.assertEquals("response ID mismatch", responseId, request.getResponseId()); ++responseId; org.apache.hadoop.yarn.api.records.Token yarnToken = null; if (amToken != null) { yarnToken = org.apache.hadoop.yarn.api.records.Token.newInstance( amToken.getIdentifier(), amToken.getKind().toString(), amToken.getPassword(), amToken.getService().toString()); } AllocateResponse response = AllocateResponse.newInstance(responseId, Collections.<ContainerStatus>emptyList(), Collections.<Container>emptyList(), Collections.<NodeReport>emptyList(), Resources.none(), null, 1, null, Collections.<NMToken>emptyList(), yarnToken, Collections.<UpdatedContainer>emptyList()); response.setApplicationPriority(Priority.newInstance(0)); return response; }