@Test public void testEmptyWorks() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); MessageCodec cmc = new MessageCodec(); Codec.Encoder encoder = cmc.getEncoder(dos); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(0, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = cmc.getDecoder(dis); assertFalse(decoder.advance()); dis.close(); assertEquals(0, cis.getCount()); }
@Test public void testOne() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); MessageCodec cmc = new MessageCodec(); Codec.Encoder encoder = cmc.getEncoder(dos); final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v")); encoder.write(kv); encoder.flush(); dos.close(); long offset = cos.getCount(); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = cmc.getDecoder(dis); assertTrue(decoder.advance()); // First read should pull in the KV assertFalse(decoder.advance()); // Second read should trip over the end-of-stream marker and return false dis.close(); assertEquals(offset, cis.getCount()); }
@Test public void testEmptyWorks() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); KeyValueCodec kvc = new KeyValueCodec(); Codec.Encoder encoder = kvc.getEncoder(dos); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(0, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = kvc.getDecoder(dis); assertFalse(decoder.advance()); dis.close(); assertEquals(0, cis.getCount()); }
@Test public void testOne() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); KeyValueCodec kvc = new KeyValueCodec(); Codec.Encoder encoder = kvc.getEncoder(dos); final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v")); final long length = kv.getLength() + Bytes.SIZEOF_INT; encoder.write(kv); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(length, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = kvc.getDecoder(dis); assertTrue(decoder.advance()); // First read should pull in the KV // Second read should trip over the end-of-stream marker and return false assertFalse(decoder.advance()); dis.close(); assertEquals(length, cis.getCount()); }
@Test public void testEmptyWorks() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodec(); Codec.Encoder encoder = codec.getEncoder(dos); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(0, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertFalse(decoder.advance()); dis.close(); assertEquals(0, cis.getCount()); }
@Test public void testOne() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodec(); Codec.Encoder encoder = codec.getEncoder(dos); final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v")); kv.setSequenceId(Long.MAX_VALUE); encoder.write(kv); encoder.flush(); dos.close(); long offset = cos.getCount(); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertTrue(decoder.advance()); // First read should pull in the KV // Second read should trip over the end-of-stream marker and return false assertFalse(decoder.advance()); dis.close(); assertEquals(offset, cis.getCount()); }
@Test public void encodeDecodeTest() throws IOException { ArthmeticCoder.SimpleFrequency freq = new ArthmeticCoder.SimpleFrequency(counts); ByteArrayOutputStream encodedPool = new ByteArrayOutputStream(); CountingOutputStream outputCounting = new CountingOutputStream(encodedPool); ArthmeticCoder.Encoder encoder = new ArthmeticCoder.Encoder(freq, new BitWrappedOutputStream(outputCounting)); for (int s : symbols) { encoder.write(s); } encoder.seal(); ByteArrayInputStream decodedPool = new ByteArrayInputStream(encodedPool.toByteArray()); CountingInputStream inputCounting = new CountingInputStream(decodedPool); ArthmeticCoder.Decoder decoder = new ArthmeticCoder.Decoder(freq, new BitWrappedInputStream(inputCounting)); int[] symbols2 = new int[symbols.length]; for (int i = 0; i < symbols.length; i++) { symbols2[i] = decoder.read(); } Assert.assertEquals(outputCounting.getCount(), inputCounting.getCount()); Assert.assertArrayEquals(symbols, symbols2); }
/** * A utility method that passes the given (unencoded) elements through * coder's registerByteSizeObserver() and encode() methods, and confirms * they are mutually consistent. This is useful for testing coder * implementations. */ public static <T> void testByteCount(Coder<T> coder, Coder.Context context, T[] elements) throws Exception { TestElementByteSizeObserver observer = new TestElementByteSizeObserver(); try (CountingOutputStream os = new CountingOutputStream(ByteStreams.nullOutputStream())) { for (T elem : elements) { coder.registerByteSizeObserver(elem, observer); coder.encode(elem, os, context); observer.advance(); } long expectedLength = os.getCount(); if (!context.isWholeStream) { assertEquals(expectedLength, observer.getSum()); } assertEquals(elements.length, observer.getCount()); } }
/** * Calculate the serialized object size in bytes. This is a good indication of how much memory the object roughly takes. * Note that this is typically not exactly the same for many objects. * @param object any object that implements {@link Serializable} * @return number of bytes of the serialized object. */ public long byteCount(Object object) { try { CountingOutputStream cos = new CountingOutputStream(new OutputStream() { @Override public void write(int b) throws IOException { // do nothing } }); ObjectOutputStream os = new ObjectOutputStream(cos); os.writeObject(object); os.flush(); os.close(); return cos.getCount(); } catch (IOException e) { throw new IllegalStateException("error serializing object: " + e.getMessage(),e); } }
public PayloadRecorder ( final boolean autoFinish ) throws IOException { this.autoFinish = autoFinish; this.tempFile = Files.createTempFile ( "rpm-", null ); try { this.fileStream = new BufferedOutputStream ( Files.newOutputStream ( this.tempFile, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING ) ); this.payloadCounter = new CountingOutputStream ( this.fileStream ); final GZIPOutputStream payloadStream = new GZIPOutputStream ( this.payloadCounter ); this.archiveCounter = new CountingOutputStream ( payloadStream ); // setup archive stream this.archiveStream = new CpioArchiveOutputStream ( this.archiveCounter, CpioConstants.FORMAT_NEW, 4, "UTF-8" ); } catch ( final IOException e ) { Files.deleteIfExists ( this.tempFile ); throw e; } }
@Test public void testOne() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodec(); Codec.Encoder encoder = codec.getEncoder(dos); final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v")); kv.setMvccVersion(Long.MAX_VALUE); encoder.write(kv); encoder.flush(); dos.close(); long offset = cos.getCount(); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertTrue(decoder.advance()); // First read should pull in the KV // Second read should trip over the end-of-stream marker and return false assertFalse(decoder.advance()); dis.close(); assertEquals(offset, cis.getCount()); }
private synchronized void openNewFile() throws IOException { if(currentFile != null) { // flush and automatically closes file try(OutputStream out = this.currentFile) { out.flush(); } } long timestamp = System.currentTimeMillis(); long count = 0; Path filePath; do { String file = String.format("crawl_data-%d-%d.deflate", timestamp, count++); filePath = directory.resolve(file); } while (Files.exists(filePath)); OutputStream fileStream = new PrintStream(filePath.toFile()); this.bytesCounter = new CountingOutputStream(fileStream); this.currentFile = new DeflaterOutputStream(this.bytesCounter, true); }
private long write(String type, Copier copier) throws IOException { synchronized (lock) { if (closed) { return -1; } long startTick = ticker.read(); out.startBlock(); NonClosingCountingOutputStream countingStreamAfterCompression = new NonClosingCountingOutputStream(out); CountingOutputStream countingStreamBeforeCompression = new CountingOutputStream(newLZFOutputStream(countingStreamAfterCompression)); copier.copyTo(countingStreamBeforeCompression); countingStreamBeforeCompression.close(); long endTick = ticker.read(); CappedDatabaseStats stats = statsByType.get(type); if (stats == null) { stats = new CappedDatabaseStats(); statsByType.put(type, stats); } stats.record(countingStreamBeforeCompression.getCount(), countingStreamAfterCompression.getCount(), endTick - startTick); return out.endBlock(); } }
/** * Creates a reference FASTA file writer. * <p> * You can specify a specific path for the index and dictionary file. If either set to {@code null} such * a file won't be generated. * </p> * * @param fastaFile the output fasta file path. * @param indexFile the path of the index file, if requested, {@code null} if none should be generated. * @param dictFile the path of the dictFile, if requested, {@code null} if nono should be generated. * @throws IllegalArgumentException if {@code fastaFile} is {@code null} or {@code basesPerLine} is 0 or negative. * @throws IOException if such exception is thrown when accessing the output path resources. */ public FastaReferenceWriter(final Path fastaFile, final int basesPerLine, final Path indexFile, final Path dictFile) throws IOException { // This code is a slight repeat of {@link #FastaReferenceWriter(OutputStream,int,OutputStream,OutputStream) // for the sake of avoiding creating output if basesPerLine is invalid. this.defaultBasePerLine = checkBasesPerLine(basesPerLine); this.fastaStream = new CountingOutputStream(Files.newOutputStream(Utils.nonNull(fastaFile))); this.indexWriter = indexFile == null ? new NullWriter() : new OutputStreamWriter(Files.newOutputStream(indexFile), CHARSET); final BufferedWriter dictWriter = new BufferedWriter(dictFile == null ? new NullWriter() : new OutputStreamWriter(Files.newOutputStream(dictFile), CHARSET)); this.dictWriter = dictWriter; this.dictCodec = new SAMSequenceDictionaryCodec(dictWriter); this.dictCodec.encodeHeaderLine(false); this.sequenceNamesAndSizes = new LinkedHashMap<>(); }
@Test public void testOne() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); Codec codec = new CellCodec(); Codec.Encoder encoder = codec.getEncoder(dos); final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v")); encoder.write(kv); encoder.flush(); dos.close(); long offset = cos.getCount(); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = codec.getDecoder(dis); assertTrue(decoder.advance()); // First read should pull in the KV // Second read should trip over the end-of-stream marker and return false assertFalse(decoder.advance()); dis.close(); assertEquals(offset, cis.getCount()); }
/** * Creates a term dictionary with the given terms. * * @param terms the terms. */ public FieldTermsDictionary(Dictionary terms) { this.terms = checkNotNull(terms); // A hack to estimate the memory size of the dictionary long bytes = -1; try (CountingOutputStream cos = new CountingOutputStream(ByteStreams.nullOutputStream()); ObjectOutputStream oos = new ObjectOutputStream(cos)) { oos.writeObject(terms); oos.flush(); bytes = cos.getCount(); } catch (IOException e) { // ignore } this.estimatedMemorySize = bytes; }
/** * Test if addStdoutOutput() actually works, and the output can be correctly captured. * * @throws InterruptedException * @throws ExecutionException * @throws IOException */ @Test public void testStdout() throws InterruptedException, ExecutionException, IOException { FFmpegBuilder builder = new FFmpegBuilder() .setInput(Samples.big_buck_bunny_720p_1mb) .addStdoutOutput() .setFormat("s8") .setAudioChannels(1) .done(); List<String> newArgs = ImmutableList.<String>builder().add(ffmpeg.getPath()).addAll(builder.build()).build(); // TODO Add support to the FFmpegJob to export the stream Process p = new ProcessBuilder(newArgs).start(); CountingOutputStream out = new CountingOutputStream(ByteStreams.nullOutputStream()); ByteStreams.copy(p.getInputStream(), out); assertEquals(0, p.waitFor()); // This is perhaps fragile, but one byte per audio sample assertEquals(254976, out.getCount()); }
/** * Starts writing to the given offset. Can be beyond the current length of the file. */ public DataOutputStream start(long offset) throws IOException { file.seek(offset); bufferedOutputStream.clear(); countingOutputStream = new CountingOutputStream(bufferedOutputStream); return new DataOutputStream(countingOutputStream); }
