private void incrementAllocation(ReservationAllocation reservation) { assert (readWriteLock.isWriteLockedByCurrentThread()); Map<ReservationInterval, ReservationRequest> allocationRequests = reservation.getAllocationRequests(); // check if we have encountered the user earlier and if not add an entry String user = reservation.getUser(); RLESparseResourceAllocation resAlloc = userResourceAlloc.get(user); if (resAlloc == null) { resAlloc = new RLESparseResourceAllocation(resCalc, minAlloc); userResourceAlloc.put(user, resAlloc); } for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests .entrySet()) { resAlloc.addInterval(r.getKey(), r.getValue()); rleSparseVector.addInterval(r.getKey(), r.getValue()); } }
private void validateInput(Plan plan, ReservationRequest rr, Resource totalCapacity) throws ContractValidationException { if (rr.getConcurrency() < 1) { throw new ContractValidationException("Gang Size should be >= 1"); } if (rr.getNumContainers() <= 0) { throw new ContractValidationException("Num containers should be >= 0"); } // check that gangSize and numContainers are compatible if (rr.getNumContainers() % rr.getConcurrency() != 0) { throw new ContractValidationException( "Parallelism must be an exact multiple of gang size"); } // check that the largest container request does not exceed // the cluster-wide limit for container sizes if (Resources.greaterThan(plan.getResourceCalculator(), totalCapacity, rr.getCapability(), plan.getMaximumAllocation())) { throw new ContractValidationException("Individual" + " capability requests should not exceed cluster's maxAlloc"); } }
InMemoryReservationAllocation(ReservationId reservationID, ReservationDefinition contract, String user, String planName, long startTime, long endTime, Map<ReservationInterval, ReservationRequest> allocationRequests, ResourceCalculator calculator, Resource minAlloc) { this.contract = contract; this.startTime = startTime; this.endTime = endTime; this.reservationID = reservationID; this.user = user; this.allocationRequests = allocationRequests; this.planName = planName; resourcesOverTime = new RLESparseResourceAllocation(calculator, minAlloc); for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests .entrySet()) { resourcesOverTime.addInterval(r.getKey(), r.getValue()); if (r.getValue().getConcurrency() > 1) { hasGang = true; } } }
@Test public void testBlocks() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testSteps() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, true, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals( Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testSkyline() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 0, 5, 10, 10, 5, 0 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, true, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals( Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testZeroAlloaction() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = {}; long start = 0; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = new HashMap<ReservationInterval, ReservationRequest>(); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, (int) start, alloc); Assert.assertFalse(rAllocation.containsGangs()); }
@Test public void testGangAlloaction() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, true); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertTrue(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i)); } }
private Map<ReservationInterval, ReservationRequest> generateAllocation( int startTime, int[] alloc, boolean isStep, boolean isGang) { Map<ReservationInterval, ReservationRequest> req = new HashMap<ReservationInterval, ReservationRequest>(); int numContainers = 0; for (int i = 0; i < alloc.length; i++) { if (isStep) { numContainers = alloc[i] + i; } else { numContainers = alloc[i]; } ReservationRequest rr = ReservationRequest.newInstance(Resource.newInstance(1024, 1, 1), (numContainers)); if (isGang) { rr.setConcurrency(numContainers); } req.put(new ReservationInterval(startTime + i, startTime + i + 1), rr); } return req; }
private void prepareBasicPlan() throws PlanningException { // insert in the reservation a couple of controlled reservations, to create // conditions for assignment that are non-empty int[] f = { 10, 10, 20, 20, 20, 10, 10 }; assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation( ReservationSystemTestUtil.getNewReservationId(), null, "u1", "dedicated", 0L, 0L + f.length * step, ReservationSystemTestUtil .generateAllocation(0, step, f), res, minAlloc))); int[] f2 = { 5, 5, 5, 5, 5, 5, 5 }; Map<ReservationInterval, ReservationRequest> alloc = ReservationSystemTestUtil.generateAllocation(5000, step, f2); assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation( ReservationSystemTestUtil.getNewReservationId(), null, "u1", "dedicated", 5000, 5000 + f2.length * step, alloc, res, minAlloc))); System.out.println("--------BEFORE AGENT----------"); System.out.println(plan.toString()); System.out.println(plan.toCumulativeString()); }
@Test public void testAddEmptyReservation() { Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, true); ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); int[] alloc = {}; int start = 100; Map<ReservationInterval, ReservationRequest> allocations = new HashMap<ReservationInterval, ReservationRequest>(); ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length, alloc.length, allocations.values()); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length, allocations, resCalc, minAlloc); Assert.assertNull(plan.getReservationById(reservationID)); try { plan.addReservation(rAllocation); } catch (PlanningException e) { Assert.fail(e.getMessage()); } }
private Map<ReservationInterval, ReservationRequest> generateAllocation( int startTime, int[] alloc, boolean isStep) { Map<ReservationInterval, ReservationRequest> req = new HashMap<ReservationInterval, ReservationRequest>(); int numContainers = 0; for (int i = 0; i < alloc.length; i++) { if (isStep) { numContainers = alloc[i] + i; } else { numContainers = alloc[i]; } ReservationRequest rr = ReservationRequest.newInstance(Resource.newInstance(1024, 1, 1), (numContainers)); req.put(new ReservationInterval(startTime + i, startTime + i + 1), rr); } return req; }
@Test public void testFailAvgBySum() throws IOException, PlanningException { // generate an allocation which violates the 25% average by sum Map<ReservationInterval, ReservationRequest> req = new TreeMap<ReservationInterval, ReservationRequest>(); long win = 86400000 / 4 + 1; int cont = (int) Math.ceil(0.5 * totCont); req.put(new ReservationInterval(initTime, initTime + win), ReservationRequest.newInstance(Resource.newInstance(1024, 1, 1), cont)); assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation( ReservationSystemTestUtil.getNewReservationId(), null, "u1", "dedicated", initTime, initTime + win, req, res, minAlloc))); try { assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation( ReservationSystemTestUtil.getNewReservationId(), null, "u1", "dedicated", initTime, initTime + win, req, res, minAlloc))); Assert.fail("should not have accepted this"); } catch (PlanningQuotaException e) { // expected } }
private Map<ReservationInterval, ReservationRequest> generateAllocation( int startTime, int[] alloc, boolean isStep) { Map<ReservationInterval, ReservationRequest> req = new HashMap<ReservationInterval, ReservationRequest>(); int numContainers = 0; for (int i = 0; i < alloc.length; i++) { if (isStep) { numContainers = alloc[i] + i; } else { numContainers = alloc[i]; } req.put(new ReservationInterval(startTime + i, startTime + i + 1), ReservationRequest.newInstance(Resource.newInstance(1024, 1, 1), (numContainers))); } return req; }
private ReservationSubmissionRequest createSimpleReservationSubmissionRequest( int numRequests, int numContainers, long arrival, long deadline, long duration) { // create a request with a single atomic ask ReservationSubmissionRequest request = new ReservationSubmissionRequestPBImpl(); ReservationDefinition rDef = new ReservationDefinitionPBImpl(); rDef.setArrival(arrival); rDef.setDeadline(deadline); if (numRequests > 0) { ReservationRequests reqs = new ReservationRequestsPBImpl(); rDef.setReservationRequests(reqs); if (numContainers > 0) { ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1, 1), numContainers, 1, duration); reqs.setReservationResources(Collections.singletonList(r)); reqs.setInterpreter(ReservationRequestInterpreter.R_ALL); } } request.setQueue(PLAN_NAME); request.setReservationDefinition(rDef); return request; }
private ReservationUpdateRequest createSimpleReservationUpdateRequest( int numRequests, int numContainers, long arrival, long deadline, long duration) { // create a request with a single atomic ask ReservationUpdateRequest request = new ReservationUpdateRequestPBImpl(); ReservationDefinition rDef = new ReservationDefinitionPBImpl(); rDef.setArrival(arrival); rDef.setDeadline(deadline); if (numRequests > 0) { ReservationRequests reqs = new ReservationRequestsPBImpl(); rDef.setReservationRequests(reqs); if (numContainers > 0) { ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1, 1), numContainers, 1, duration); reqs.setReservationResources(Collections.singletonList(r)); reqs.setInterpreter(ReservationRequestInterpreter.R_ALL); } } request.setReservationDefinition(rDef); request.setReservationId(ReservationSystemTestUtil.getNewReservationId()); return request; }
private ReservationSubmissionRequest createSimpleReservationRequest( int numContainers, long arrival, long deadline, long duration) { // create a request with a single atomic ask ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1, 1), numContainers, 1, duration); ReservationRequests reqs = ReservationRequests.newInstance(Collections.singletonList(r), ReservationRequestInterpreter.R_ALL); ReservationDefinition rDef = ReservationDefinition.newInstance(arrival, deadline, reqs, "testClientRMService#reservation"); ReservationSubmissionRequest request = ReservationSubmissionRequest.newInstance(rDef, ReservationSystemTestUtil.reservationQ); return request; }
private ReservationSubmissionRequest createSimpleReservationRequest( int numContainers, long arrival, long deadline, long duration) { // create a request with a single atomic ask ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1), numContainers, 1, duration); ReservationRequests reqs = ReservationRequests.newInstance(Collections.singletonList(r), ReservationRequestInterpreter.R_ALL); ReservationDefinition rDef = ReservationDefinition.newInstance(arrival, deadline, reqs, "testYarnClient#reservation"); ReservationSubmissionRequest request = ReservationSubmissionRequest.newInstance(rDef, ReservationSystemTestUtil.reservationQ); return request; }
private ReservationSubmissionRequest createSimpleReservationRequest( int numContainers, long arrival, long deadline, long duration) { // create a request with a single atomic ask ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1), numContainers, 1, duration); ReservationRequests reqs = ReservationRequests.newInstance(Collections.singletonList(r), ReservationRequestInterpreter.R_ALL); ReservationDefinition rDef = ReservationDefinition.newInstance(arrival, deadline, reqs, "testClientRMService#reservation"); ReservationSubmissionRequest request = ReservationSubmissionRequest.newInstance(rDef, ReservationSystemTestUtil.reservationQ); return request; }
private Map<ReservationInterval, Resource> generateAllocation( int startTime, int[] alloc, boolean isStep, boolean isGang) { Map<ReservationInterval, Resource> req = new HashMap<ReservationInterval, Resource>(); int numContainers = 0; for (int i = 0; i < alloc.length; i++) { if (isStep) { numContainers = alloc[i] + i; } else { numContainers = alloc[i]; } ReservationRequest rr = ReservationRequest.newInstance(Resource.newInstance(1024, 1), (numContainers)); if (isGang) { rr.setConcurrency(numContainers); } req.put(new ReservationInterval(startTime + i, startTime + i + 1), ReservationSystemUtil.toResource(rr)); } return req; }
public static ReservationSubmissionRequest createSimpleReservationRequest( int numContainers, long arrival, long deadline, long duration) { // create a request with a single atomic ask ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1), numContainers, 1, duration); ReservationRequests reqs = ReservationRequests.newInstance(Collections.singletonList(r), ReservationRequestInterpreter.R_ALL); ReservationDefinition rDef = ReservationDefinition.newInstance(arrival, deadline, reqs, "testClientRMService#reservation"); ReservationSubmissionRequest request = ReservationSubmissionRequest.newInstance(rDef, reservationQ); return request; }
private ReservationUpdateRequest createSimpleReservationUpdateRequest( int numRequests, int numContainers, long arrival, long deadline, long duration) { // create a request with a single atomic ask ReservationUpdateRequest request = new ReservationUpdateRequestPBImpl(); ReservationDefinition rDef = new ReservationDefinitionPBImpl(); rDef.setArrival(arrival); rDef.setDeadline(deadline); if (numRequests > 0) { ReservationRequests reqs = new ReservationRequestsPBImpl(); rDef.setReservationRequests(reqs); if (numContainers > 0) { ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1), numContainers, 1, duration); reqs.setReservationResources(Collections.singletonList(r)); reqs.setInterpreter(ReservationRequestInterpreter.R_ALL); } } request.setReservationDefinition(rDef); request.setReservationId(ReservationSystemTestUtil.getNewReservationId()); return request; }
private Map<ReservationInterval, ReservationRequest> generateAllocation( int startTime, int[] alloc, boolean isStep) { Map<ReservationInterval, ReservationRequest> req = new HashMap<ReservationInterval, ReservationRequest>(); int numContainers = 0; for (int i = 0; i < alloc.length; i++) { if (isStep) { numContainers = alloc[i] + i; } else { numContainers = alloc[i]; } ReservationRequest rr = ReservationRequest.newInstance(Resource.newInstance(1024, 1), (numContainers)); req.put(new ReservationInterval(startTime + i, startTime + i + 1), rr); } return req; }
@Test(expected = PlanningQuotaException.class) public void testFailAvg() throws IOException, PlanningException { // generate an allocation which violates the 25% average single-shot Map<ReservationInterval, Resource> req = new TreeMap<ReservationInterval, Resource>(); long win = timeWindow / 2 + 100; int cont = (int) Math.ceil(0.5 * totCont); req.put(new ReservationInterval(initTime, initTime + win), ReservationSystemUtil.toResource( ReservationRequest.newInstance(Resource.newInstance(1024, 1), cont))); ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition( initTime, initTime + win, win); assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation( ReservationSystemTestUtil.getNewReservationId(), rDef, "u1", "dedicated", initTime, initTime + win, req, res, minAlloc), false)); }
private Map<ReservationInterval, Resource> generateAllocation(int startTime, int[] alloc, boolean isStep) { Map<ReservationInterval, Resource> req = new HashMap<ReservationInterval, Resource>(); int numContainers = 0; for (int i = 0; i < alloc.length; i++) { if (isStep) { numContainers = alloc[i] + i; } else { numContainers = alloc[i]; } req.put(new ReservationInterval(startTime + i, startTime + i + 1), ReservationSystemUtil.toResource(ReservationRequest.newInstance( Resource.newInstance(1024, 1), (numContainers)))); } return req; }
private ReservationSubmissionRequest createSimpleReservationSubmissionRequest( int numRequests, int numContainers, long arrival, long deadline, long duration) { // create a request with a single atomic ask ReservationSubmissionRequest request = new ReservationSubmissionRequestPBImpl(); ReservationDefinition rDef = new ReservationDefinitionPBImpl(); rDef.setArrival(arrival); rDef.setDeadline(deadline); if (numRequests > 0) { ReservationRequests reqs = new ReservationRequestsPBImpl(); rDef.setReservationRequests(reqs); if (numContainers > 0) { ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1), numContainers, 1, duration); reqs.setReservationResources(Collections.singletonList(r)); reqs.setInterpreter(ReservationRequestInterpreter.R_ALL); } } request.setQueue(PLAN_NAME); request.setReservationDefinition(rDef); return request; }
@Test public void testBlocks() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testSteps() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, true, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals( Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testSkyline() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 0, 5, 10, 10, 5, 0 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, true, false); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertFalse(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals( Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i)); } }
@Test public void testGangAlloaction() { ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong()); int[] alloc = { 10, 10, 10, 10, 10, 10 }; int start = 100; ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length); Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, true); ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc); doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc); Assert.assertTrue(rAllocation.containsGangs()); for (int i = 0; i < alloc.length; i++) { Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i)); } }
private Map<ReservationInterval, ReservationRequest> generateAllocation( int startTime, int[] alloc, boolean isStep, boolean isGang) { Map<ReservationInterval, ReservationRequest> req = new HashMap<ReservationInterval, ReservationRequest>(); int numContainers = 0; for (int i = 0; i < alloc.length; i++) { if (isStep) { numContainers = alloc[i] + i; } else { numContainers = alloc[i]; } ReservationRequest rr = ReservationRequest.newInstance(Resource.newInstance(1024, 1), (numContainers)); if (isGang) { rr.setConcurrency(numContainers); } req.put(new ReservationInterval(startTime + i, startTime + i + 1), rr); } return req; }