@Test public void tesAllJobs() throws Exception { final ApplicationClientProtocol applicationsManager = Mockito.mock(ApplicationClientProtocol.class); GetApplicationsResponse allApplicationsResponse = Records .newRecord(GetApplicationsResponse.class); List<ApplicationReport> applications = new ArrayList<ApplicationReport>(); applications.add(getApplicationReport(YarnApplicationState.FINISHED, FinalApplicationStatus.FAILED)); applications.add(getApplicationReport(YarnApplicationState.FINISHED, FinalApplicationStatus.SUCCEEDED)); applications.add(getApplicationReport(YarnApplicationState.FINISHED, FinalApplicationStatus.KILLED)); applications.add(getApplicationReport(YarnApplicationState.FAILED, FinalApplicationStatus.FAILED)); allApplicationsResponse.setApplicationList(applications); Mockito.when( applicationsManager.getApplications(Mockito .any(GetApplicationsRequest.class))).thenReturn( allApplicationsResponse); ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate( new YarnConfiguration()) { @Override protected void serviceStart() throws Exception { Assert.assertTrue(this.client instanceof YarnClientImpl); ((YarnClientImpl) this.client).setRMClient(applicationsManager); } }; JobStatus[] allJobs = resourceMgrDelegate.getAllJobs(); Assert.assertEquals(State.FAILED, allJobs[0].getState()); Assert.assertEquals(State.SUCCEEDED, allJobs[1].getState()); Assert.assertEquals(State.KILLED, allJobs[2].getState()); Assert.assertEquals(State.FAILED, allJobs[3].getState()); }
@Override public GetApplicationsResponse getApplications(GetApplicationsRequest request) throws YarnException, IOException { long startedBegin = request.getStartRange() == null ? 0L : request.getStartRange() .getMinimumLong(); long startedEnd = request.getStartRange() == null ? Long.MAX_VALUE : request .getStartRange().getMaximumLong(); GetApplicationsResponse response = GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>( history.getApplications(request.getLimit(), startedBegin, startedEnd) .values())); return response; }
protected List<ApplicationReport> fetchAppReports() throws YarnException, IOException { List<ApplicationReport> ret; EnumSet<YarnApplicationState> states = EnumSet.of(YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING); GetApplicationsRequest req = GetApplicationsRequest.newInstance(types, states); req.setQueues(queues); req.setUsers(users); ret = applicationReportsCache.getIfPresent(req); if (ret != null) { return ret; } ret = client.getApplications(queues, users, types, states); applicationReportsCache.put(req, ret); return ret; }
@Test public void testApplications() throws IOException, YarnException { ApplicationId appId = null; appId = ApplicationId.newInstance(0, 1); ApplicationId appId1 = ApplicationId.newInstance(0, 2); GetApplicationsRequest request = GetApplicationsRequest.newInstance(); GetApplicationsResponse response = clientService.getClientHandler().getApplications(request); List<ApplicationReport> appReport = response.getApplicationList(); Assert.assertNotNull(appReport); Collections.sort(appReport, new Comparator<ApplicationReport>() { @Override public int compare(ApplicationReport o1, ApplicationReport o2) { return o1.getApplicationId().compareTo(o2.getApplicationId()); } }); Assert.assertEquals(appId, appReport.get(0).getApplicationId()); Assert.assertEquals(appId1, appReport.get(1).getApplicationId()); }
private GetApplicationsResponse getApplicationReports( List<ApplicationReport> applicationReports, GetApplicationsRequest request) { List<ApplicationReport> appReports = new ArrayList<ApplicationReport>(); Set<String> appTypes = request.getApplicationTypes(); boolean bypassFilter = appTypes.isEmpty(); for (ApplicationReport appReport : applicationReports) { if (!(bypassFilter || appTypes.contains( appReport.getApplicationType()))) { continue; } appReports.add(appReport); } GetApplicationsResponse response = GetApplicationsResponse.newInstance(appReports); return response; }
@Test public void testLocalMode() throws IOException, YarnException, InterpreterException { InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark"); sparkInterpreterSetting.setProperty("master", "local[*]"); sparkInterpreterSetting.setProperty("SPARK_HOME", System.getenv("SPARK_HOME")); sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath()); sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false"); sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false"); testInterpreterBasics(); // no yarn application launched GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING)); GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request); assertEquals(0, response.getApplicationList().size()); interpreterSettingManager.close(); }
@Test public void testYarnClientMode() throws IOException, YarnException, InterruptedException, InterpreterException { InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark"); sparkInterpreterSetting.setProperty("master", "yarn-client"); sparkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath()); sparkInterpreterSetting.setProperty("SPARK_HOME", System.getenv("SPARK_HOME")); sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath()); sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false"); sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false"); sparkInterpreterSetting.setProperty("PYSPARK_PYTHON", getPythonExec()); sparkInterpreterSetting.setProperty("spark.driver.memory", "512m"); testInterpreterBasics(); // 1 yarn application launched GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING)); GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request); assertEquals(1, response.getApplicationList().size()); interpreterSettingManager.close(); }
@Test public void testYarnClusterMode() throws IOException, YarnException, InterruptedException, InterpreterException { InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark"); sparkInterpreterSetting.setProperty("master", "yarn-cluster"); sparkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath()); sparkInterpreterSetting.setProperty("SPARK_HOME", System.getenv("SPARK_HOME")); sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath()); sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false"); sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false"); sparkInterpreterSetting.setProperty("spark.pyspark.python", getPythonExec()); sparkInterpreterSetting.setProperty("spark.driver.memory", "512m"); testInterpreterBasics(); // 1 yarn application launched GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING)); GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request); assertEquals(1, response.getApplicationList().size()); interpreterSettingManager.close(); }
@Test public void testApplications() throws IOException, YarnException { ApplicationId appId = null; appId = ApplicationId.newInstance(0, 1); writeApplicationStartData(appId); writeApplicationFinishData(appId); ApplicationId appId1 = ApplicationId.newInstance(0, 2); writeApplicationStartData(appId1); writeApplicationFinishData(appId1); GetApplicationsRequest request = GetApplicationsRequest.newInstance(); GetApplicationsResponse response = historyServer.getClientService().getClientHandler() .getApplications(request); List<ApplicationReport> appReport = response.getApplicationList(); Assert.assertNotNull(appReport); Assert.assertEquals(appId, appReport.get(0).getApplicationId()); Assert.assertEquals(appId1, appReport.get(1).getApplicationId()); }
@Override public GetApplicationsResponse getApplications(GetApplicationsRequest request) throws YarnException, IOException { GetApplicationsRequestProto requestProto = ((GetApplicationsRequestPBImpl) request).getProto(); try { return new GetApplicationsResponsePBImpl(proxy.getApplications(null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); return null; } }
@Override public GetApplicationsResponse getApplications( GetApplicationsRequest request) throws YarnException, IOException { GetApplicationsRequestProto requestProto = ((GetApplicationsRequestPBImpl) request).getProto(); try { return new GetApplicationsResponsePBImpl(proxy.getApplications( null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); return null; } }
@Test public void testAppsRace() throws Exception { // mock up an RM that returns app reports for apps that don't exist // in the RMApps list ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationReport mockReport = mock(ApplicationReport.class); when(mockReport.getApplicationId()).thenReturn(appId); GetApplicationsResponse mockAppsResponse = mock(GetApplicationsResponse.class); when(mockAppsResponse.getApplicationList()) .thenReturn(Arrays.asList(new ApplicationReport[] { mockReport })); ClientRMService mockClientSvc = mock(ClientRMService.class); when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class), anyBoolean())).thenReturn(mockAppsResponse); ResourceManager mockRM = mock(ResourceManager.class); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, null, null, null, null); when(mockRM.getRMContext()).thenReturn(rmContext); when(mockRM.getClientRMService()).thenReturn(mockClientSvc); RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(), mock(HttpServletResponse.class)); final Set<String> emptySet = Collections.unmodifiableSet(Collections.<String>emptySet()); // verify we don't get any apps when querying HttpServletRequest mockHsr = mock(HttpServletRequest.class); AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null, null, null, null, null, null, null, null, emptySet, emptySet); assertTrue(appsInfo.getApps().isEmpty()); // verify we don't get an NPE when specifying a final status query appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED", null, null, null, null, null, null, null, emptySet, emptySet); assertTrue(appsInfo.getApps().isEmpty()); }
public static ClientRMService mockClientRMService(RMContext rmContext) { ClientRMService clientRMService = mock(ClientRMService.class); List<ApplicationReport> appReports = new ArrayList<ApplicationReport>(); for (RMApp app : rmContext.getRMApps().values()) { ApplicationReport appReport = ApplicationReport.newInstance( app.getApplicationId(), (ApplicationAttemptId) null, app.getUser(), app.getQueue(), app.getName(), (String) null, 0, (Token) null, app.createApplicationState(), app.getDiagnostics().toString(), (String) null, app.getStartTime(), app.getFinishTime(), app.getFinalApplicationStatus(), (ApplicationResourceUsageReport) null, app.getTrackingUrl(), app.getProgress(), app.getApplicationType(), (Token) null); appReports.add(appReport); } GetApplicationsResponse response = mock(GetApplicationsResponse.class); when(response.getApplicationList()).thenReturn(appReports); try { when(clientRMService.getApplications(any(GetApplicationsRequest.class))) .thenReturn(response); } catch (YarnException e) { Assert.fail("Exception is not expteced."); } return clientRMService; }
private void verifyOwnerAccess() throws Exception { AccessControlList viewACL = new AccessControlList(""); viewACL.addGroup(FRIENDLY_GROUP); AccessControlList modifyACL = new AccessControlList(""); modifyACL.addUser(FRIEND); ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL); final GetApplicationReportRequest appReportRequest = recordFactory .newRecordInstance(GetApplicationReportRequest.class); appReportRequest.setApplicationId(applicationId); final KillApplicationRequest finishAppRequest = recordFactory .newRecordInstance(KillApplicationRequest.class); finishAppRequest.setApplicationId(applicationId); // View as owner rmClient.getApplicationReport(appReportRequest); // List apps as owner Assert.assertEquals("App view by owner should list the apps!!", 1, rmClient.getApplications( recordFactory.newRecordInstance(GetApplicationsRequest.class)) .getApplicationList().size()); // Kill app as owner rmClient.forceKillApplication(finishAppRequest); resourceManager.waitForState(applicationId, RMAppState.KILLED); }
private void verifySuperUserAccess() throws Exception { AccessControlList viewACL = new AccessControlList(""); viewACL.addGroup(FRIENDLY_GROUP); AccessControlList modifyACL = new AccessControlList(""); modifyACL.addUser(FRIEND); ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL); final GetApplicationReportRequest appReportRequest = recordFactory .newRecordInstance(GetApplicationReportRequest.class); appReportRequest.setApplicationId(applicationId); final KillApplicationRequest finishAppRequest = recordFactory .newRecordInstance(KillApplicationRequest.class); finishAppRequest.setApplicationId(applicationId); ApplicationClientProtocol superUserClient = getRMClientForUser(SUPER_USER); // View as the superUser superUserClient.getApplicationReport(appReportRequest); // List apps as superUser Assert.assertEquals("App view by super-user should list the apps!!", 2, superUserClient.getApplications( recordFactory.newRecordInstance(GetApplicationsRequest.class)) .getApplicationList().size()); // Kill app as the superUser superUserClient.forceKillApplication(finishAppRequest); resourceManager.waitForState(applicationId, RMAppState.KILLED); }
private void verifyFriendAccess() throws Exception { AccessControlList viewACL = new AccessControlList(""); viewACL.addGroup(FRIENDLY_GROUP); AccessControlList modifyACL = new AccessControlList(""); modifyACL.addUser(FRIEND); ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL); final GetApplicationReportRequest appReportRequest = recordFactory .newRecordInstance(GetApplicationReportRequest.class); appReportRequest.setApplicationId(applicationId); final KillApplicationRequest finishAppRequest = recordFactory .newRecordInstance(KillApplicationRequest.class); finishAppRequest.setApplicationId(applicationId); ApplicationClientProtocol friendClient = getRMClientForUser(FRIEND); // View as the friend friendClient.getApplicationReport(appReportRequest); // List apps as friend Assert.assertEquals("App view by a friend should list the apps!!", 3, friendClient.getApplications( recordFactory.newRecordInstance(GetApplicationsRequest.class)) .getApplicationList().size()); // Kill app as the friend friendClient.forceKillApplication(finishAppRequest); resourceManager.waitForState(applicationId, RMAppState.KILLED); }
private void verifyAdministerQueueUserAccess() throws Exception { isQueueUser = true; AccessControlList viewACL = new AccessControlList(""); viewACL.addGroup(FRIENDLY_GROUP); AccessControlList modifyACL = new AccessControlList(""); modifyACL.addUser(FRIEND); ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL); final GetApplicationReportRequest appReportRequest = recordFactory .newRecordInstance(GetApplicationReportRequest.class); appReportRequest.setApplicationId(applicationId); final KillApplicationRequest finishAppRequest = recordFactory .newRecordInstance(KillApplicationRequest.class); finishAppRequest.setApplicationId(applicationId); ApplicationClientProtocol administerQueueUserRmClient = getRMClientForUser(QUEUE_ADMIN_USER); // View as the administerQueueUserRmClient administerQueueUserRmClient.getApplicationReport(appReportRequest); // List apps as administerQueueUserRmClient Assert.assertEquals("App view by queue-admin-user should list the apps!!", 5, administerQueueUserRmClient.getApplications( recordFactory.newRecordInstance(GetApplicationsRequest.class)) .getApplicationList().size()); // Kill app as the administerQueueUserRmClient administerQueueUserRmClient.forceKillApplication(finishAppRequest); resourceManager.waitForState(applicationId, RMAppState.KILLED); }
@Override public GetApplicationsResponse getApplications(GetApplicationsRequest request) throws YarnException, IOException { GetApplicationsResponse response = GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>( history.getAllApplications().values())); return response; }
@Test public void testApplications() throws IOException, YarnException { ApplicationId appId = null; appId = ApplicationId.newInstance(0, 1); ApplicationId appId1 = ApplicationId.newInstance(0, 2); GetApplicationsRequest request = GetApplicationsRequest.newInstance(); GetApplicationsResponse response = clientService.getApplications(request); List<ApplicationReport> appReport = response.getApplicationList(); Assert.assertNotNull(appReport); Assert.assertEquals(appId, appReport.get(0).getApplicationId()); Assert.assertEquals(appId1, appReport.get(1).getApplicationId()); }
@Override public List<ApplicationReport> getApplications() throws YarnException, IOException { GetApplicationsRequest request = GetApplicationsRequest.newInstance(null, null); GetApplicationsResponse response = ahsClient.getApplications(request); return response.getApplicationList(); }
@Override public List<ApplicationReport> getApplications(Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates) throws YarnException, IOException { GetApplicationsRequest request = GetApplicationsRequest.newInstance(applicationTypes, applicationStates); GetApplicationsResponse response = rmClient.getApplications(request); return response.getApplicationList(); }
@Override public GetApplicationsResponse getApplications( GetApplicationsRequest request) throws YarnException { resetStartFailoverFlag(true); // make sure failover has been triggered Assert.assertTrue(waittingForFailOver()); // create GetApplicationsResponse with fake applicationList GetApplicationsResponse response = GetApplicationsResponse.newInstance(createFakeAppReports()); return response; }
@Test public void testAppsRace() throws Exception { // mock up an RM that returns app reports for apps that don't exist // in the RMApps list ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationReport mockReport = mock(ApplicationReport.class); when(mockReport.getApplicationId()).thenReturn(appId); GetApplicationsResponse mockAppsResponse = mock(GetApplicationsResponse.class); when(mockAppsResponse.getApplicationList()) .thenReturn(Arrays.asList(new ApplicationReport[] { mockReport })); ClientRMService mockClientSvc = mock(ClientRMService.class); when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class), anyBoolean())).thenReturn(mockAppsResponse); ResourceManager mockRM = mock(ResourceManager.class); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, null, null, null, null); when(mockRM.getRMContext()).thenReturn(rmContext); when(mockRM.getClientRMService()).thenReturn(mockClientSvc); rmContext.setNodeLabelManager(mock(RMNodeLabelsManager.class)); RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(), mock(HttpServletResponse.class)); final Set<String> emptySet = Collections.unmodifiableSet(Collections.<String>emptySet()); // verify we don't get any apps when querying HttpServletRequest mockHsr = mock(HttpServletRequest.class); AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null, null, null, null, null, null, null, null, emptySet, emptySet); assertTrue(appsInfo.getApps().isEmpty()); // verify we don't get an NPE when specifying a final status query appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED", null, null, null, null, null, null, null, emptySet, emptySet); assertTrue(appsInfo.getApps().isEmpty()); }
@Test public void testApplications() throws IOException, YarnException { ApplicationId appId = null; appId = ApplicationId.newInstance(0, 1); ApplicationId appId1 = ApplicationId.newInstance(0, 2); GetApplicationsRequest request = GetApplicationsRequest.newInstance(); GetApplicationsResponse response = clientService.getApplications(request); List<ApplicationReport> appReport = response.getApplicationList(); Assert.assertNotNull(appReport); Assert.assertEquals(appId, appReport.get(1).getApplicationId()); Assert.assertEquals(appId1, appReport.get(0).getApplicationId()); // Create a historyManager, and set the max_apps can be loaded // as 1. Configuration conf = new YarnConfiguration(); conf.setLong(YarnConfiguration.APPLICATION_HISTORY_MAX_APPS, 1); ApplicationHistoryManagerOnTimelineStore historyManager2 = new ApplicationHistoryManagerOnTimelineStore(dataManager, new ApplicationACLsManager(conf)); historyManager2.init(conf); historyManager2.start(); @SuppressWarnings("resource") ApplicationHistoryClientService clientService2 = new ApplicationHistoryClientService(historyManager2); response = clientService2.getApplications(request); appReport = response.getApplicationList(); Assert.assertNotNull(appReport); Assert.assertTrue(appReport.size() == 1); // Expected to get the appReport for application with appId1 Assert.assertEquals(appId1, appReport.get(0).getApplicationId()); }
@Override public List<ApplicationReport> getApplications(Set<String> queues, Set<String> users, Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates) throws YarnException, IOException { GetApplicationsRequest request = GetApplicationsRequest.newInstance(applicationTypes, applicationStates); request.setQueues(queues); request.setUsers(users); GetApplicationsResponse response = rmClient.getApplications(request); return response.getApplicationList(); }
