public IndexJsonSink(Index index, EventHandler eventHandler) { this.index = index; this.eventHandler = eventHandler; this.out = new FileBackedOutputStream(1024 * 1024); this.npw = new OutputStreamJsonSink(out); }
/** * To get around the size limit, you could use FileBackedOutputStream to store * some in memory and some on disk. * <p> * On T3500 Machine, 22.808 seconds for 1gb payload, no caching, fbos(1Mb) * </p> * <p> * Sends output to nullOutputStream to avoid crashing eclipse. * </p> * <p> * Note: This is also generating the payload from scratch (twice, * duplicateInput), but this operation is better than having the payload in * your repo and needing to download on checkout. * </p> * <p> * Also has potential for crashing due to disk space. * </p> */ @Test public void storingOnDiskIsSlowAndCanCrash() throws IOException { FileBackedOutputStream outputToInput = new FileBackedOutputStream(mb1); duplicateInput(Res.randomBytesOfLength(gb1), outputToInput); Processor.run(outputToInput.asByteSource(), ByteStreams.nullOutputStream()); }
/** * This one caches the payload to disk before duplicating to show that * regenerating the payload is faster than reading directly from disk. * <p> * On T3500 Machine, 31.692 seconds for 1gb payload, with caching, fbos(1Mb) * <p> * <p> * Caching: The randomly generated input is cached to disk so that when the * duplicateInput() method is called, the random generation doesnt have to run * again. * </p> * <p> * Sends output to nullOutputStream to avoid crashing eclipse. * </p> * <p> * Note: This is also generating the payload from scratch (twice), but this * operation is better than having the payload in your repo and needing to * download on checkout. * </p> */ @Test public void cachedStoringOnDiskIsSlow() throws IOException { FileBackedOutputStream outputToInput = new FileBackedOutputStream(mb1); duplicateInput(Res.randomCachedBytesOfLength(gb1), outputToInput); Processor.run(outputToInput.asByteSource(), ByteStreams.nullOutputStream()); }