/** * Find the split points for a given sample. The sample keys are sorted * and down sampled to find even split points for the partitions. The * returned keys should be the start of their respective partitions. * @param numPartitions the desired number of partitions * @return an array of size numPartitions - 1 that holds the split points */ Text[] createPartitions(int numPartitions) { int numRecords = records.size(); System.out.println("Making " + numPartitions + " from " + numRecords + " sampled records"); if (numPartitions > numRecords) { throw new IllegalArgumentException ("Requested more partitions than input keys (" + numPartitions + " > " + numRecords + ")"); } new QuickSort().sort(this, 0, records.size()); float stepSize = numRecords / (float) numPartitions; Text[] result = new Text[numPartitions-1]; for(int i=1; i < numPartitions; ++i) { result[i-1] = records.get(Math.round(stepSize * i)); } return result; }
/** * Find the split points for a given sample. The sample keys are sorted * and down sampled to find even split points for the partitions. The * returned keys should be the start of their respective partitions. * @param numPartitions the desired number of partitions * @return an array of size numPartitions - 1 that holds the split points */ Text[] createPartitions(int numPartitions) { int numRecords = records.size(); System.out.println("Making " + numPartitions + " from " + numRecords + " records"); if (numPartitions > numRecords) { throw new IllegalArgumentException ("Requested more partitions than input keys (" + numPartitions + " > " + numRecords + ")"); } new QuickSort().sort(this, 0, records.size()); float stepSize = numRecords / (float) numPartitions; System.out.println("Step size is " + stepSize); Text[] result = new Text[numPartitions-1]; for(int i=1; i < numPartitions; ++i) { result[i-1] = records.get(Math.round(stepSize * i)); } return result; }
/** * Find the split points for a given sample. The sample keys are sorted * and down sampled to find even split points for the partitions. The * returned keys should be the start of their respective partitions. * @param numPartitions the desired number of partitions * @return an array of size numPartitions - 1 that holds the split points */ public ArrayList<WritableComparable> createPartitions(int numPartitions) { int numRecords = records.size(); if (numPartitions > numRecords) { throw new IllegalArgumentException ("Requested more partitions than input keys (" + numPartitions + " > " + numRecords + ")"); } new QuickSort().sort(this, 0, records.size()); //System.out.println("after sort: "+ toString()); float stepSize = numRecords / (float) numPartitions; //System.out.println("Step size is " + stepSize); ArrayList<WritableComparable> result = new ArrayList<>(numPartitions-1); for(int i=1; i < numPartitions; i++) { result.add(records.get(Math.round(stepSize * i))); } return result; }
Text[] createPartitions(int numPartitions) { int numRecords = this.records.size(); System.out.println("Making " + numPartitions + " from " + numRecords + " records"); if(numPartitions > numRecords) { throw new IllegalArgumentException("Requested more partitions than input keys (" + numPartitions + " > " + numRecords + ")"); } else { (new QuickSort()).sort(this, 0, this.records.size()); float stepSize = (float)numRecords / (float)numPartitions; System.out.println("Step size is " + stepSize); Text[] result = new Text[numPartitions - 1]; for(int i = 1; i < numPartitions; ++i) { result[i - 1] = (Text)this.records.get(Math.round(stepSize * (float)i)); } return result; } }
/** * Find the split points for a given sample. The sample keys are sorted * and down sampled to find even split points for the partitions. The * returned keys should be the start of their respective partitions. * * @param numPartitions the desired number of partitions * @return an array of size numPartitions - 1 that holds the split * points */ Text[] createPartitions(int numPartitions) { int numRecords = records.size(); System.out.println("Making " + numPartitions + " from " + numRecords + " records"); if (numPartitions > numRecords) { throw new IllegalArgumentException("Requested more partitions than input keys (" + numPartitions + " > " + numRecords + ")"); } new QuickSort().sort(this, 0, records.size()); float stepSize = numRecords / (float) numPartitions; System.out.println("Step size is " + stepSize); Text[] result = new Text[numPartitions - 1]; for (int i = 1; i < numPartitions; ++i) { result[i - 1] = records.get(Math.round(stepSize * i)); } return result; }
public void initialize(JobConf job, TaskReporter reporter, TaskAttemptID taskId) throws ClassNotFoundException, IOException{ this.reporter = reporter; this.taskId = taskId; mapOutputByteCounter = reporter.getCounter(MAP_OUTPUT_BYTES); mapOutputRecordCounter = reporter.getCounter(MAP_OUTPUT_RECORDS); this.job = job; sorter = ReflectionUtils.newInstance( job.getClass("map.sort.class", QuickSort.class, IndexedSorter.class), job); partitions = job.getNumReduceTasks(); if (partitionInd == null || partitions * 2 != partitionInd.length) { partitionInd = new int[partitions * 2]; } comparator = job.getOutputKeyComparator(); keyClass = (Class)job.getMapOutputKeyClass(); valClass = (Class)job.getMapOutputValueClass(); serializationFactory = new SerializationFactory(job); keySerializer = serializationFactory.getSerializer(keyClass); keySerializer.open(bb); valSerializer = serializationFactory.getSerializer(valClass); valSerializer.open(bb); reset(); }
@Override public void sort(SelectionVector4 vector4, VectorContainer container){ Stopwatch watch = new Stopwatch(); watch.start(); QuickSort qs = new QuickSort(); qs.sort(this, 0, vector4.getTotalCount()); logger.debug("Took {} us to sort {} records", watch.elapsed(TimeUnit.MICROSECONDS), vector4.getTotalCount()); }
@Override public void sort(SelectionVector2 vector2){ QuickSort qs = new QuickSort(); Stopwatch watch = new Stopwatch(); watch.start(); if (vector2.getCount() > 0) { qs.sort(this, 0, vector2.getCount()); } logger.debug("Took {} us to sort {} records", watch.elapsed(TimeUnit.MICROSECONDS), vector2.getCount()); }
public long doSort(){ QuickSort qs = new QuickSort(); ByteSortable b = new ByteSortable(); long nano = System.nanoTime(); qs.sort(b, 0, RECORD_COUNT); return System.nanoTime() - nano; }
@Override public void sort(SelectionVector2 vector2){ QuickSort qs = new QuickSort(); Stopwatch watch = Stopwatch.createStarted(); if (vector2.getCount() > 0) { qs.sort(this, 0, vector2.getCount()); } logger.debug("Took {} us to sort {} records", watch.elapsed(TimeUnit.MICROSECONDS), vector2.getCount()); }
@Override public void sort(SelectionVector4 vector4, VectorContainer container){ Stopwatch watch = Stopwatch.createStarted(); QuickSort qs = new QuickSort(); qs.sort(this, 0, vector4.getTotalCount()); logger.debug("Took {} us to sort {} records", watch.elapsed(TimeUnit.MICROSECONDS), vector4.getTotalCount()); }
protected void sortMemBlock(MemoryBlock memBlock) { if (memBlock.currentPtr <= 0) { return; } // quick sort the offsets OffsetSortable sortableObj = new OffsetSortable(memBlock, kvbuffer); QuickSort quickSort = new QuickSort(); quickSort.sort(sortableObj, 0, memBlock.currentPtr); }
