private void initNodeIds(int nodeNums, int containerNums, Resource resource) { Random portRandom = new Random(); Random ipRandom = new Random(); for (int i = 0; i < nodeNums; i++) { NodeReport nodeReport = Records.newRecord(NodeReport.class); nodeReport.setNumContainers(containerNums); nodeReport.setNodeLabels(new HashSet<>()); nodeReport.setNodeState(NodeState.RUNNING); nodeReport.setCapability(resource); nodeReport.setUsed(Resource.newInstance(0, 0)); int port = 1024 + portRandom.nextInt(65535 - 1024 + 1); StringBuilder hostStr = new StringBuilder(); for (int j = 0; j < 4; j++) { hostStr.append(".").append(ipRandom.nextInt(256)); } NodeId nodeId = NodeId.newInstance(hostStr.substring(1), port); nodeReport.setNodeId(nodeId); nodeReport.setHttpAddress(nodeId.getHost()); nodeReportList.add(nodeReport); } }
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState, String httpAddress, String rackName, Resource used, Resource capability, int numContainers, String healthReport, long lastHealthReportTime, Set<String> nodeLabels) { NodeReport nodeReport = recordFactory.newRecordInstance(NodeReport.class); nodeReport.setNodeId(nodeId); nodeReport.setNodeState(nodeState); nodeReport.setHttpAddress(httpAddress); nodeReport.setRackName(rackName); nodeReport.setUsed(used); nodeReport.setCapability(capability); nodeReport.setNumContainers(numContainers); nodeReport.setHealthReport(healthReport); nodeReport.setLastHealthReportTime(lastHealthReportTime); nodeReport.setNodeLabels(nodeLabels); return nodeReport; }
public static AllocateResponse newAllocateResponse(int responseId, List<ContainerStatus> completedContainers, List<Container> allocatedContainers, List<NodeReport> updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt) { AllocateResponse response = recordFactory .newRecordInstance(AllocateResponse.class); response.setNumClusterNodes(numClusterNodes); response.setResponseId(responseId); response.setCompletedContainersStatuses(completedContainers); response.setAllocatedContainers(allocatedContainers); response.setUpdatedNodes(updatedNodes); response.setAvailableResources(availResources); response.setAMCommand(command); response.setPreemptionMessage(preempt); return response; }
@Override public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request) throws YarnException { GetClusterNodesResponse response = recordFactory.newRecordInstance(GetClusterNodesResponse.class); EnumSet<NodeState> nodeStates = request.getNodeStates(); if (nodeStates == null || nodeStates.isEmpty()) { nodeStates = EnumSet.allOf(NodeState.class); } Collection<RMNode> nodes = RMServerUtils.queryRMNodes(rmContext, nodeStates); List<NodeReport> nodeReports = new ArrayList<NodeReport>(nodes.size()); for (RMNode nodeInfo : nodes) { nodeReports.add(createNodeReports(nodeInfo)); } response.setNodeReports(nodeReports); return response; }
private NodeReport createNodeReports(RMNode rmNode) { SchedulerNodeReport schedulerNodeReport = scheduler.getNodeReport(rmNode.getNodeID()); Resource used = BuilderUtils.newResource(0, 0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.getUsedResource(); numContainers = schedulerNodeReport.getNumContainers(); } NodeReport report = BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(), rmNode.getHttpAddress(), rmNode.getRackName(), used, rmNode.getTotalCapability(), numContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels()); return report; }
/** * Lists the nodes matching the given node states * * @param nodeStates * @throws YarnException * @throws IOException */ private void listClusterNodes(Set<NodeState> nodeStates) throws YarnException, IOException { PrintWriter writer = new PrintWriter( new OutputStreamWriter(sysout, Charset.forName("UTF-8"))); List<NodeReport> nodesReport = client.getNodeReports( nodeStates.toArray(new NodeState[0])); writer.println("Total Nodes:" + nodesReport.size()); writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address", "Number-of-Running-Containers"); for (NodeReport nodeReport : nodesReport) { writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport .getNodeState(), nodeReport.getHttpAddress(), nodeReport .getNumContainers()); } writer.flush(); }
private List<NodeReport> getNodeReports(int noOfNodes, NodeState state, boolean emptyNodeLabel) { List<NodeReport> nodeReports = new ArrayList<NodeReport>(); for (int i = 0; i < noOfNodes; i++) { Set<String> nodeLabels = null; if (!emptyNodeLabel) { // node labels is not ordered, but when we output it, it should be // ordered nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y"); } NodeReport nodeReport = NodeReport.newInstance(NodeId .newInstance("host" + i, 0), state, "host" + 1 + ":8888", "rack1", Records.newRecord(Resource.class), Records .newRecord(Resource.class), 0, "", 0, nodeLabels); nodeReports.add(nodeReport); } return nodeReports; }
@Public @Stable public static AllocateResponse newInstance(int responseId, List<ContainerStatus> completedContainers, List<Container> allocatedContainers, List<NodeReport> updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt, List<NMToken> nmTokens) { AllocateResponse response = Records.newRecord(AllocateResponse.class); response.setNumClusterNodes(numClusterNodes); response.setResponseId(responseId); response.setCompletedContainersStatuses(completedContainers); response.setAllocatedContainers(allocatedContainers); response.setUpdatedNodes(updatedNodes); response.setAvailableResources(availResources); response.setAMCommand(command); response.setPreemptionMessage(preempt); response.setNMTokens(nmTokens); return response; }
@Public @Stable public static AllocateResponse newInstance(int responseId, List<ContainerStatus> completedContainers, List<Container> allocatedContainers, List<NodeReport> updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt, List<NMToken> nmTokens, List<ContainerResourceIncrease> increasedContainers, List<ContainerResourceDecrease> decreasedContainers) { AllocateResponse response = newInstance(responseId, completedContainers, allocatedContainers, updatedNodes, availResources, command, numClusterNodes, preempt, nmTokens); response.setIncreasedContainers(increasedContainers); response.setDecreasedContainers(decreasedContainers); return response; }
@Private @Unstable public static AllocateResponse newInstance(int responseId, List<ContainerStatus> completedContainers, List<Container> allocatedContainers, List<NodeReport> updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken, List<ContainerResourceIncrease> increasedContainers, List<ContainerResourceDecrease> decreasedContainers) { AllocateResponse response = newInstance(responseId, completedContainers, allocatedContainers, updatedNodes, availResources, command, numClusterNodes, preempt, nmTokens, increasedContainers, decreasedContainers); response.setAMRMToken(amRMToken); return response; }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { Assert.assertEquals("response ID mismatch", responseId, request.getResponseId()); ++responseId; org.apache.hadoop.yarn.api.records.Token yarnToken = null; if (amToken != null) { yarnToken = org.apache.hadoop.yarn.api.records.Token.newInstance( amToken.getIdentifier(), amToken.getKind().toString(), amToken.getPassword(), amToken.getService().toString()); } return AllocateResponse.newInstance(responseId, Collections.<ContainerStatus>emptyList(), Collections.<Container>emptyList(), Collections.<NodeReport>emptyList(), Resources.none(), null, 1, null, Collections.<NMToken>emptyList(), yarnToken, Collections.<ContainerResourceIncrease>emptyList(), Collections.<ContainerResourceDecrease>emptyList()); }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { lastAsk = request.getAskList(); for (ResourceRequest req : lastAsk) { if (ResourceRequest.ANY.equals(req.getResourceName())) { Priority priority = req.getPriority(); if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) { lastAnyAskMap = req.getNumContainers(); } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){ lastAnyAskReduce = req.getNumContainers(); } } } AllocateResponse response = AllocateResponse.newInstance( request.getResponseId(), containersToComplete, containersToAllocate, Collections.<NodeReport>emptyList(), Resource.newInstance(512000, 1024, 1024), null, 10, null, Collections.<NMToken>emptyList()); containersToComplete.clear(); containersToAllocate.clear(); return response; }
@Override public List<NodeReport> getClusterNodes(EnumSet<NodeState> states) throws IOException { String sinfoCmd = conf.get( HPCConfiguration.YARN_APPLICATION_HPC_COMMAND_SLURM_SINFO, HPCConfiguration.DEFAULT_YARN_APPLICATION_HPC_COMMAND_SLURM_SINFO); String result = Shell.execCommand(sinfoCmd, "-h", "-o %N"); List<NodeReport> reports = new ArrayList<NodeReport>(); for (String node : HPCUtils.parseHostList(result)) { NodeReport nodeReport = NodeReport.newInstance( NodeId.newInstance(node, 0), NodeState.RUNNING, "", "", Resource.newInstance(0, 0), Resource.newInstance(1024, 2), 0, "", 0); reports.add(nodeReport); } return reports; }
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState, String httpAddress, String rackName, Resource used, Resource capability, int numContainers, String healthReport, long lastHealthReportTime, Set<String> nodeLabels, ResourceUtilization containersUtilization, ResourceUtilization nodeUtilization) { NodeReport nodeReport = recordFactory.newRecordInstance(NodeReport.class); nodeReport.setNodeId(nodeId); nodeReport.setNodeState(nodeState); nodeReport.setHttpAddress(httpAddress); nodeReport.setRackName(rackName); nodeReport.setUsed(used); nodeReport.setCapability(capability); nodeReport.setNumContainers(numContainers); nodeReport.setHealthReport(healthReport); nodeReport.setLastHealthReportTime(lastHealthReportTime); nodeReport.setNodeLabels(nodeLabels); nodeReport.setAggregatedContainersUtilization(containersUtilization); nodeReport.setNodeUtilization(nodeUtilization); return nodeReport; }
private NodeReport createNodeReports(RMNode rmNode) { SchedulerNodeReport schedulerNodeReport = scheduler.getNodeReport(rmNode.getNodeID()); Resource used = BuilderUtils.newResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.getUsedResource(); numContainers = schedulerNodeReport.getNumContainers(); } NodeReport report = BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(), rmNode.getHttpAddress(), rmNode.getRackName(), used, rmNode.getTotalCapability(), numContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(), rmNode.getNodeUtilization()); return report; }
@Public @Stable public static AllocateResponse newInstance(int responseId, List<ContainerStatus> completedContainers, List<Container> allocatedContainers, List<NodeReport> updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt, List<NMToken> nmTokens, List<Container> increasedContainers, List<Container> decreasedContainers) { AllocateResponse response = newInstance(responseId, completedContainers, allocatedContainers, updatedNodes, availResources, command, numClusterNodes, preempt, nmTokens); response.setIncreasedContainers(increasedContainers); response.setDecreasedContainers(decreasedContainers); return response; }
@Private @Unstable public static AllocateResponse newInstance(int responseId, List<ContainerStatus> completedContainers, List<Container> allocatedContainers, List<NodeReport> updatedNodes, Resource availResources, AMCommand command, int numClusterNodes, PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken, List<Container> increasedContainers, List<Container> decreasedContainers) { AllocateResponse response = newInstance(responseId, completedContainers, allocatedContainers, updatedNodes, availResources, command, numClusterNodes, preempt, nmTokens, increasedContainers, decreasedContainers); response.setAMRMToken(amRMToken); return response; }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { Assert.assertEquals("response ID mismatch", responseId, request.getResponseId()); ++responseId; org.apache.hadoop.yarn.api.records.Token yarnToken = null; if (amToken != null) { yarnToken = org.apache.hadoop.yarn.api.records.Token.newInstance( amToken.getIdentifier(), amToken.getKind().toString(), amToken.getPassword(), amToken.getService().toString()); } AllocateResponse response = AllocateResponse.newInstance(responseId, Collections.<ContainerStatus>emptyList(), Collections.<Container>emptyList(), Collections.<NodeReport>emptyList(), Resources.none(), null, 1, null, Collections.<NMToken>emptyList(), yarnToken, Collections.<Container>emptyList(), Collections.<Container>emptyList()); response.setApplicationPriority(Priority.newInstance(0)); return response; }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { lastAsk = request.getAskList(); for (ResourceRequest req : lastAsk) { if (ResourceRequest.ANY.equals(req.getResourceName())) { Priority priority = req.getPriority(); if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) { lastAnyAskMap = req.getNumContainers(); } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){ lastAnyAskReduce = req.getNumContainers(); } } } AllocateResponse response = AllocateResponse.newInstance( request.getResponseId(), containersToComplete, containersToAllocate, Collections.<NodeReport>emptyList(), Resource.newInstance(512000, 1024), null, 10, null, Collections.<NMToken>emptyList()); // RM will always ensure that a default priority is sent to AM response.setApplicationPriority(Priority.newInstance(0)); containersToComplete.clear(); containersToAllocate.clear(); return response; }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { lastAsk = request.getAskList(); for (ResourceRequest req : lastAsk) { if (ResourceRequest.ANY.equals(req.getResourceName())) { Priority priority = req.getPriority(); if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) { lastAnyAskMap = req.getNumContainers(); } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){ lastAnyAskReduce = req.getNumContainers(); } } } AllocateResponse response = AllocateResponse.newInstance( request.getResponseId(), containersToComplete, containersToAllocate, Collections.<NodeReport>emptyList(), Resource.newInstance(512000, 1024), null, 10, null, Collections.<NMToken>emptyList()); containersToComplete.clear(); containersToAllocate.clear(); return response; }
private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException { List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING); int totalFreeMemory = 0; int containerLimit = 0; int[] nodeManagersFree = new int[nodes.size()]; for (int i = 0; i < nodes.size(); i++) { NodeReport rep = nodes.get(i); int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0); nodeManagersFree[i] = free; totalFreeMemory += free; if (free > containerLimit) { containerLimit = free; } } return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree); }
private NodeReport createNodeReports(RMNode rmNode) { SchedulerNodeReport schedulerNodeReport = scheduler.getNodeReport(rmNode.getNodeID()); Resource used = BuilderUtils.newResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.getUsedResource(); numContainers = schedulerNodeReport.getNumContainers(); } NodeReport report = BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(), rmNode.getHttpAddress(), rmNode.getRackName(), used, rmNode.getTotalCapability(), numContainers, rmNode.getHealthReport(), rmNode.getLastHealthReportTime(), rmNode.getNodeLabels()); return report; }
@Override public void setNodeReports(List<NodeReport> nodeManagers) { if (nodeManagers == null) { builder.clearNodeReports(); } this.nodeManagerInfoList = nodeManagers; }
private void initLocalNodeManagerInfosList() { if (this.nodeManagerInfoList != null) { return; } GetClusterNodesResponseProtoOrBuilder p = viaProto ? proto : builder; List<NodeReportProto> list = p.getNodeReportsList(); nodeManagerInfoList = new ArrayList<NodeReport>(); for (NodeReportProto a : list) { nodeManagerInfoList.add(convertFromProtoFormat(a)); } }
private void addLocalNodeManagerInfosToProto() { maybeInitBuilder(); builder.clearNodeReports(); if (nodeManagerInfoList == null) return; Iterable<NodeReportProto> iterable = new Iterable<NodeReportProto>() { @Override public Iterator<NodeReportProto> iterator() { return new Iterator<NodeReportProto>() { Iterator<NodeReport> iter = nodeManagerInfoList.iterator(); @Override public boolean hasNext() { return iter.hasNext(); } @Override public NodeReportProto next() { return convertToProtoFormat(iter.next()); } @Override public void remove() { throw new UnsupportedOperationException(); } }; } }; builder.addAllNodeReports(iterable); }
@Override public synchronized void setUpdatedNodes( final List<NodeReport> updatedNodes) { if (updatedNodes == null) { this.updatedNodes.clear(); return; } this.updatedNodes = new ArrayList<NodeReport>(updatedNodes.size()); this.updatedNodes.addAll(updatedNodes); }
private synchronized void initLocalNewNodeReportList() { if (this.updatedNodes != null) { return; } AllocateResponseProtoOrBuilder p = viaProto ? proto : builder; List<NodeReportProto> list = p.getUpdatedNodesList(); updatedNodes = new ArrayList<NodeReport>(list.size()); for (NodeReportProto n : list) { updatedNodes.add(convertFromProtoFormat(n)); } }
private synchronized Iterable<NodeReportProto> getNodeReportProtoIterable( final List<NodeReport> newNodeReportsList) { maybeInitBuilder(); return new Iterable<NodeReportProto>() { @Override public synchronized Iterator<NodeReportProto> iterator() { return new Iterator<NodeReportProto>() { Iterator<NodeReport> iter = newNodeReportsList.iterator(); @Override public synchronized boolean hasNext() { return iter.hasNext(); } @Override public synchronized NodeReportProto next() { return convertToProtoFormat(iter.next()); } @Override public synchronized void remove() { throw new UnsupportedOperationException(); } }; } }; }
@Override public List<NodeReport> getNodeReports(NodeState... states) throws YarnException, IOException { EnumSet<NodeState> statesSet = (states.length == 0) ? EnumSet.allOf(NodeState.class) : EnumSet.noneOf(NodeState.class); for (NodeState state : states) { statesSet.add(state); } GetClusterNodesRequest request = GetClusterNodesRequest .newInstance(statesSet); GetClusterNodesResponse response = rmClient.getClusterNodes(request); return response.getNodeReports(); }
