@Override public YarnClusterMetrics getClusterMetrics() throws IOException { int noOfNodes = 0; String pbsNodesCmd = conf.get( HPCConfiguration.YARN_APPLICATION_HPC_COMMAND_PBS_PBSNODES, HPCConfiguration.DEFAULT_YARN_APPLICATION_HPC_COMMAND_PBS_PBSNODES); String result = Shell.execCommand(pbsNodesCmd, "-a"); String[] resultLines = result.split("\\n"); for (String line : resultLines) { Matcher matcher = Pattern.compile("(\\s+.+)").matcher(line); if (!matcher.find()) { noOfNodes++; } } return YarnClusterMetrics.newInstance(noOfNodes); }
@Override public GetClusterMetricsResponse getClusterMetrics( GetClusterMetricsRequest request) throws YarnException { GetClusterMetricsResponse response = recordFactory .newRecordInstance(GetClusterMetricsResponse.class); YarnClusterMetrics ymetrics = recordFactory .newRecordInstance(YarnClusterMetrics.class); ymetrics.setNumNodeManagers(this.rmContext.getRMNodes().size()); ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics(); ymetrics.setNumDecommissionedNodeManagers(clusterMetrics .getNumDecommisionedNMs()); ymetrics.setNumActiveNodeManagers(clusterMetrics.getNumActiveNMs()); ymetrics.setNumLostNodeManagers(clusterMetrics.getNumLostNMs()); ymetrics.setNumUnhealthyNodeManagers(clusterMetrics.getUnhealthyNMs()); ymetrics.setNumRebootedNodeManagers(clusterMetrics.getNumRebootedNMs()); response.setClusterMetrics(ymetrics); return response; }
protected NodesInformation getNodesInfo() { NodesInformation nodeInfo = new NodesInformation(); YarnClusterMetrics yarnClusterMetrics; try { yarnClusterMetrics = client.getYarnClusterMetrics(); } catch (IOException ie) { LOG.error("Unable to fetch cluster metrics", ie); return nodeInfo; } catch (YarnException ye) { LOG.error("Unable to fetch cluster metrics", ye); return nodeInfo; } nodeInfo.decommissionedNodes = yarnClusterMetrics.getNumDecommissionedNodeManagers(); nodeInfo.totalNodes = yarnClusterMetrics.getNumNodeManagers(); nodeInfo.runningNodes = yarnClusterMetrics.getNumActiveNodeManagers(); nodeInfo.lostNodes = yarnClusterMetrics.getNumLostNodeManagers(); nodeInfo.unhealthyNodes = yarnClusterMetrics.getNumUnhealthyNodeManagers(); nodeInfo.rebootedNodes = yarnClusterMetrics.getNumRebootedNodeManagers(); return nodeInfo; }
public boolean run() throws YarnException, IOException { yarnClient.start(); YarnClientApplication client = yarnClient.createApplication(); GetNewApplicationResponse appResponse = client.getNewApplicationResponse(); appId = appResponse.getApplicationId(); LOG.info("Applicatoin ID = {}", appId); int maxMemory = appResponse.getMaximumResourceCapability().getMemory(); int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max memory = {} and max vcores = {}", maxMemory, maxVCores); YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); LOG.info("Number of NodeManagers = {}", clusterMetrics.getNumNodeManagers()); List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); for (NodeReport node : nodeReports) { LOG.info("Node ID = {}, address = {}, containers = {}", node.getNodeId(), node.getHttpAddress(), node.getNumContainers()); } List<QueueInfo> queueList = yarnClient.getAllQueues(); for (QueueInfo queue : queueList) { LOG.info("Available queue: {} with capacity {} to {}", queue.getQueueName(), queue.getCapacity(), queue.getMaximumCapacity()); } return true; }
@Override public YarnClusterMetrics getClusterMetrics() { GetClusterMetricsResponseProtoOrBuilder p = viaProto ? proto : builder; if (this.yarnClusterMetrics != null) { return this.yarnClusterMetrics; } if (!p.hasClusterMetrics()) { return null; } this.yarnClusterMetrics = convertFromProtoFormat(p.getClusterMetrics()); return this.yarnClusterMetrics; }
@Override public void setClusterMetrics(YarnClusterMetrics clusterMetrics) { maybeInitBuilder(); if (clusterMetrics == null) builder.clearClusterMetrics(); this.yarnClusterMetrics = clusterMetrics; }
@Override public GetClusterMetricsResponse getClusterMetrics( GetClusterMetricsRequest request) throws YarnException { GetClusterMetricsResponse response = recordFactory .newRecordInstance(GetClusterMetricsResponse.class); YarnClusterMetrics ymetrics = recordFactory .newRecordInstance(YarnClusterMetrics.class); ymetrics.setNumNodeManagers(this.rmContext.getRMNodes().size()); response.setClusterMetrics(ymetrics); return response; }
@Override public YarnClusterMetrics getYarnClusterMetrics() throws YarnException, IOException { GetClusterMetricsRequest request = Records.newRecord(GetClusterMetricsRequest.class); GetClusterMetricsResponse response = rmClient.getClusterMetrics(request); return response.getClusterMetrics(); }
@Test(timeout = 15000) public void testGetClusterMetricsOnHA() throws Exception { YarnClusterMetrics clusterMetrics = client.getYarnClusterMetrics(); Assert.assertTrue(clusterMetrics != null); Assert.assertEquals(cluster.createFakeYarnClusterMetrics(), clusterMetrics); }
@Private @Unstable public static GetClusterMetricsResponse newInstance(YarnClusterMetrics metrics) { GetClusterMetricsResponse response = Records.newRecord(GetClusterMetricsResponse.class); response.setClusterMetrics(metrics); return response; }
public ClusterMetrics getClusterMetrics() throws IOException, InterruptedException { try { YarnClusterMetrics metrics = client.getYarnClusterMetrics(); ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1, metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1, metrics.getNumNodeManagers(), 0, 0); return oldMetrics; } catch (YarnException e) { throw new IOException(e); } }
@Override public YarnClusterMetrics getClusterMetrics() throws IOException { String sinfoCmd = conf.get( HPCConfiguration.YARN_APPLICATION_HPC_COMMAND_SLURM_SINFO, HPCConfiguration.DEFAULT_YARN_APPLICATION_HPC_COMMAND_SLURM_SINFO); String result = Shell.execCommand(sinfoCmd, "-h", "-o %D"); int parseInt; try { parseInt = Integer.parseInt(result.trim()); } catch (Throwable e) { throw new IOException("Failed to get cluster metrics", e); } return YarnClusterMetrics.newInstance(parseInt); }
@Override public String getClusterDescription() { try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos); YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics(); ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers()); List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING); final String format = "|%-16s |%-16s %n"; ps.printf("|Property |Value %n"); ps.println("+---------------------------------------+"); int totalMemory = 0; int totalCores = 0; for (NodeReport rep : nodes) { final Resource res = rep.getCapability(); totalMemory += res.getMemory(); totalCores += res.getVirtualCores(); ps.format(format, "NodeID", rep.getNodeId()); ps.format(format, "Memory", res.getMemory() + " MB"); ps.format(format, "vCores", res.getVirtualCores()); ps.format(format, "HealthReport", rep.getHealthReport()); ps.format(format, "Containers", rep.getNumContainers()); ps.println("+---------------------------------------+"); } ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores); List<QueueInfo> qInfo = yarnClient.getAllQueues(); for (QueueInfo q : qInfo) { ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " + q.getMaximumCapacity() + " Applications: " + q.getApplications().size()); } return baos.toString(); } catch (Exception e) { throw new RuntimeException("Couldn't get cluster description", e); } }
