@Override public ReservationId getReservationId() { ReservationSubmissionResponseProtoOrBuilder p = viaProto ? proto : builder; if (reservationId != null) { return reservationId; } if (!p.hasReservationId()) { return null; } reservationId = convertFromProtoFormat(p.getReservationId()); return reservationId; }
@Override public boolean deleteReservation(ReservationId reservationID) { writeLock.lock(); try { ReservationAllocation reservation = getReservationById(reservationID); if (reservation == null) { String errMsg = "The specified Reservation with ID " + reservationID + " does not exist in the plan"; LOG.error(errMsg); throw new IllegalArgumentException(errMsg); } return removeReservation(reservation); } finally { writeLock.unlock(); } }
InMemoryReservationAllocation(ReservationId reservationID, ReservationDefinition contract, String user, String planName, long startTime, long endTime, Map<ReservationInterval, ReservationRequest> allocationRequests, ResourceCalculator calculator, Resource minAlloc) { this.contract = contract; this.startTime = startTime; this.endTime = endTime; this.reservationID = reservationID; this.user = user; this.allocationRequests = allocationRequests; this.planName = planName; resourcesOverTime = new RLESparseResourceAllocation(calculator, minAlloc); for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests .entrySet()) { resourcesOverTime.addInterval(r.getKey(), r.getValue()); if (r.getValue().getConcurrency() > 1) { hasGang = true; } } }
@Test public void testBlocks() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testSteps() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, true, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals( Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testSkyline() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 0, 5, 10, 10, 5, 0 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, true, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals( Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testZeroAlloaction() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = {}; long start = 0; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = new HashMap<ReservationInterval, ReservationRequest>(); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, (int) start, alloc); Assert.assertFalse(rAllocation.containsGangs()); }
@Test public void testGangAlloaction() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, true); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertTrue(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testAddEmptyReservation() { Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, true); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); int[] alloc = {}; int start = 100; Map<ReservationInterval, ReservationRequest> allocations = new HashMap<ReservationInterval, ReservationRequest>(); ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length, alloc.length, allocations.values()); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length, allocations, resCalc, minAlloc); Assert.assertNull(plan.getReservationById(reservationID)); try { plan.addReservation(rAllocation); } catch (PlanningException e) { Assert.fail(e.getMessage()); } }
@Before public void setUp() { clock = mock(Clock.class); plan = mock(Plan.class); rSystem = mock(ReservationSystem.class); plans.put(PLAN_NAME, plan); rrValidator = new ReservationInputValidator(clock); when(clock.getTime()).thenReturn(1L); ResourceCalculator rCalc = new DefaultResourceCalculator(); Resource resource = Resource.newInstance(10240, 10, 10); when(plan.getResourceCalculator()).thenReturn(rCalc); when(plan.getTotalCapacity()).thenReturn(resource); when(rSystem.getQueueForReservation(any(ReservationId.class))).thenReturn( PLAN_NAME); when(rSystem.getPlan(PLAN_NAME)).thenReturn(plan); }
@Test public void testUpdateReservationDoesnotExist() { ReservationUpdateRequest request = createSimpleReservationUpdateRequest(1, 1, 1, 5, 4); ReservationId rId = request.getReservationId(); when(rSystem.getQueueForReservation(rId)).thenReturn(null); Plan plan = null; try { plan = rrValidator.validateReservationUpdateRequest(rSystem, request); Assert.fail(); } catch (YarnException e) { Assert.assertNull(plan); String message = e.getMessage(); Assert .assertTrue(message.equals(MessageFormat .format( "The specified reservation with ID: {0} is unknown. Please try again with a valid reservation.", rId))); LOG.info(message); } }
@Test public void testDeleteReservationDoesnotExist() { ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl(); ReservationId rId = ReservationSystemTestUtil.getNewReservationId(); request.setReservationId(rId); when(rSystem.getQueueForReservation(rId)).thenReturn(null); Plan plan = null; try { plan = rrValidator.validateReservationDeleteRequest(rSystem, request); Assert.fail(); } catch (YarnException e) { Assert.assertNull(plan); String message = e.getMessage(); Assert .assertTrue(message.equals(MessageFormat .format( "The specified reservation with ID: {0} is unknown. Please try again with a valid reservation.", rId))); LOG.info(message); } }
@Test public void testDeleteReservationInvalidPlan() { ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl(); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); request.setReservationId(reservationID); when(rSystem.getPlan(PLAN_NAME)).thenReturn(null); Plan plan = null; try { plan = rrValidator.validateReservationDeleteRequest(rSystem, request); Assert.fail(); } catch (YarnException e) { Assert.assertNull(plan); String message = e.getMessage(); Assert .assertTrue(message .endsWith(" is not associated with any valid plan. Please try again with a valid reservation.")); LOG.info(message); } }
public static InMemoryReservationAllocation toInMemoryAllocation( String planName, ReservationId reservationId, ReservationAllocationStateProto allocationState, Resource minAlloc, ResourceCalculator planResourceCalculator) { ReservationDefinition definition = convertFromProtoFormat( allocationState.getReservationDefinition()); Map<ReservationInterval, Resource> allocations = toAllocations( allocationState.getAllocationRequestsList()); InMemoryReservationAllocation allocation = new InMemoryReservationAllocation(reservationId, definition, allocationState.getUser(), planName, allocationState.getStartTime(), allocationState.getEndTime(), allocations, planResourceCalculator, minAlloc, allocationState.getContainsGangs()); return allocation; }
@Override public void recover(RMState state) throws Exception { LOG.info("Recovering Reservation system"); writeLock.lock(); try { Map<String, Map<ReservationId, ReservationAllocationStateProto>> reservationSystemState = state.getReservationState(); if (planFollower != null) { for (String plan : plans.keySet()) { // recover reservations if any from state store if (reservationSystemState.containsKey(plan)) { loadPlan(plan, reservationSystemState.get(plan)); } synchronizePlan(plan, false); } startPlanFollower(conf.getLong( YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, YarnConfiguration.DEFAULT_RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS)); } } finally { writeLock.unlock(); } }
@Test public void testSkyline() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 0, 5, 10, 10, 5, 0 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, true, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals( Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testGangAlloaction() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, true); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertTrue(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testSubmitReservation() throws JSONException, Exception { rm.start(); for (int i = 0; i < 100; i++) { MockNM amNodeManager = rm.registerNode("127.0.0." + i + ":1234", 100 * 1024); amNodeManager.nodeHeartbeat(true); } ReservationId rid = testSubmissionReservationHelper("reservation/submit", MediaType.APPLICATION_JSON); if (this.isAuthenticationEnabled()) { assertNotNull(rid); } rm.stop(); }
@Test public void testUpdateReservation() throws JSONException, Exception { rm.start(); for (int i = 0; i < 100; i++) { MockNM amNodeManager = rm.registerNode("127.0.0." + i + ":1234", 100 * 1024); amNodeManager.nodeHeartbeat(true); } ReservationId rid = testSubmissionReservationHelper("reservation/submit", MediaType.APPLICATION_JSON); if (this.isAuthenticationEnabled()) { assertNotNull(rid); } testUpdateReservationHelper("reservation/update", rid, MediaType.APPLICATION_JSON); rm.stop(); }
private void validateStoredReservation( RMStateStoreHelper stateStoreHelper, TestDispatcher dispatcher, RMContext rmContext, ReservationId r1, String planName, ReservationAllocation allocation, ReservationAllocationStateProto allocationStateProto) throws Exception { RMStateStore store = stateStoreHelper.getRMStateStore(); when(rmContext.getStateStore()).thenReturn(store); store.setRMDispatcher(dispatcher); RMState state = store.loadState(); Map<String, Map<ReservationId, ReservationAllocationStateProto>> reservationState = state.getReservationState(); Assert.assertNotNull(reservationState); Map<ReservationId, ReservationAllocationStateProto> reservations = reservationState.get(planName); Assert.assertNotNull(reservations); ReservationAllocationStateProto storedReservationAllocation = reservations.get(r1); Assert.assertNotNull(storedReservationAllocation); assertAllocationStateEqual( allocationStateProto, storedReservationAllocation); assertAllocationStateEqual(allocation, storedReservationAllocation); }
@Test public void testSteps() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition( start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, Resource> allocations = generateAllocation(start, alloc, true, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals( Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testZeroAlloaction() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = {}; long start = 0; ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition( start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, Resource> allocations = new HashMap<ReservationInterval, Resource>(); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, (int) start, alloc); Assert.assertFalse(rAllocation.containsGangs()); }
@Test public void testAddEmptyReservation() { Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, true, context); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); int[] alloc = {}; int start = 100; Map<ReservationInterval, ReservationRequest> allocations = new HashMap<ReservationInterval, ReservationRequest>(); ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length, alloc.length, allocations.values()); Map<ReservationInterval, Resource> allocs = ReservationSystemUtil.toResources (allocations); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length, allocs, resCalc, minAlloc); Assert.assertNull(plan.getReservationById(reservationID)); try { plan.addReservation(rAllocation, false); } catch (PlanningException e) { Assert.fail(e.getMessage()); } }
@Override public ReservationId getReservationId() { ReservationUpdateRequestProtoOrBuilder p = viaProto ? proto : builder; if (reservationId != null) { return reservationId; } if (!p.hasReservationId()) { return null; } reservationId = convertFromProtoFormat(p.getReservationId()); return reservationId; }
@Override public void setReservationId(ReservationId reservationId) { maybeInitBuilder(); if (reservationId == null) { builder.clearReservationId(); return; } this.reservationId = reservationId; }
@Override public ReservationId getReservationId() { ReservationDeleteRequestProtoOrBuilder p = viaProto ? proto : builder; if (reservationId != null) { return reservationId; } if (!p.hasReservationId()) { return null; } reservationId = convertFromProtoFormat(p.getReservationId()); return reservationId; }
@Override public ReservationId getReservationID() { ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; if (reservationId != null) { return reservationId; } if (!p.hasReservationId()) { return null; } reservationId = convertFromProtoFormat(p.getReservationId()); return reservationId; }
@Override public void setReservationID(ReservationId reservationID) { maybeInitBuilder(); if (reservationID == null) { builder.clearReservationId(); return; } this.reservationId = reservationID; }
@Override protected Resource getReservationQueueResourceIfExists(Plan plan, ReservationId reservationId) { CSQueue resQueue = cs.getQueue(reservationId.toString()); Resource reservationResource = null; if (resQueue != null) { reservationResource = Resources.multiply(cs.getClusterResource(), resQueue.getAbsoluteCapacity()); } return reservationResource; }
private Plan validateReservation(ReservationSystem reservationSystem, ReservationId reservationId, String auditConstant) throws YarnException { String message = ""; // check if the reservation id is valid if (reservationId == null) { message = "Missing reservation id." + " Please try again by specifying a reservation id."; RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input", "ClientRMService", message); throw RPCUtil.getRemoteException(message); } String queueName = reservationSystem.getQueueForReservation(reservationId); if (queueName == null) { message = "The specified reservation with ID: " + reservationId + " is unknown. Please try again with a valid reservation."; RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input", "ClientRMService", message); throw RPCUtil.getRemoteException(message); } // check if the associated plan is valid Plan plan = reservationSystem.getPlan(queueName); if (plan == null) { message = "The specified reservation: " + reservationId + " is not associated with any valid plan." + " Please try again with a valid reservation."; RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input", "ClientRMService", message); throw RPCUtil.getRemoteException(message); } return plan; }
/** * Quick validation on the input to check some obvious fail conditions (fail * fast) the input and returns the appropriate {@link Plan} associated with * the specified {@link Queue} or throws an exception message illustrating the * details of any validation check failures * * @param reservationSystem the {@link ReservationSystem} to validate against * @param request the {@link ReservationUpdateRequest} defining the resources * required over time for the request * @return the {@link Plan} to submit the request to * @throws YarnException */ public Plan validateReservationUpdateRequest( ReservationSystem reservationSystem, ReservationUpdateRequest request) throws YarnException { ReservationId reservationId = request.getReservationId(); Plan plan = validateReservation(reservationSystem, reservationId, AuditConstants.UPDATE_RESERVATION_REQUEST); validateReservationDefinition(reservationId, request.getReservationDefinition(), plan, AuditConstants.UPDATE_RESERVATION_REQUEST); return plan; }
@Override public ReservationAllocation getReservationById(ReservationId reservationID) { if (reservationID == null) { return null; } readLock.lock(); try { return reservationTable.get(reservationID); } finally { readLock.unlock(); } }
@Override public String getQueueForReservation(ReservationId reservationId) { readLock.lock(); try { return resQMap.get(reservationId); } finally { readLock.unlock(); } }
@Override public void setQueueForReservation(ReservationId reservationId, String queueName) { writeLock.lock(); try { resQMap.put(reservationId, queueName); } finally { writeLock.unlock(); } }