/** * Method for compressor availability check */ private static <T extends Compressor, E extends Decompressor> boolean isAvailable(TesterPair<T, E> pair) { Compressor compressor = pair.compressor; if (compressor.getClass().isAssignableFrom(Lz4Compressor.class) && (NativeCodeLoader.isNativeCodeLoaded())) return true; else if (compressor.getClass().isAssignableFrom(BuiltInZlibDeflater.class) && NativeCodeLoader.isNativeCodeLoaded()) return true; else if (compressor.getClass().isAssignableFrom(ZlibCompressor.class)) { return ZlibFactory.isNativeZlibLoaded(new Configuration()); } else if (compressor.getClass().isAssignableFrom(SnappyCompressor.class) && isNativeSnappyLoadable()) return true; return false; }
@Test public void testCompressorDecompressor() { // no more for this data int SIZE = 44 * 1024; byte[] rawData = generate(SIZE); try { CompressDecompressTester.of(rawData) .withCompressDecompressPair(new SnappyCompressor(), new SnappyDecompressor()) .withCompressDecompressPair(new Lz4Compressor(), new Lz4Decompressor()) .withCompressDecompressPair(new BuiltInZlibDeflater(), new BuiltInZlibInflater()) .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK, CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK, CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS, CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM)) .test(); } catch (Exception ex) { GenericTestUtils.assertExceptionContains( "testCompressorDecompressor error !!!", ex); } }
@Test public void testCompressorDecompressor() { // no more for this data int SIZE = 44 * 1024; byte[] rawData = generate(SIZE); try { CompressDecompressTester.of(rawData) .withCompressDecompressPair(new SnappyCompressor(), new SnappyDecompressor()) .withCompressDecompressPair(new Lz4Compressor(), new Lz4Decompressor()) .withCompressDecompressPair(new BuiltInZlibDeflater(), new BuiltInZlibInflater()) .withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK, CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK, CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS, CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM)) .test(); } catch (Exception ex) { fail("testCompressorDecompressor error !!!" + ex); } }
private void testGzipCodecWrite(boolean useNative) throws IOException { // Create a gzipped file using a compressor from the CodecPool, // and try to read it back via the regular GZIPInputStream. // Use native libs per the parameter Configuration conf = new Configuration(); if (useNative) { assumeTrue(ZlibFactory.isNativeZlibLoaded(conf)); } else { assertFalse("ZlibFactory is using native libs against request", ZlibFactory.isNativeZlibLoaded(conf)); } // Ensure that the CodecPool has a BuiltInZlibDeflater in it. Compressor zlibCompressor = ZlibFactory.getZlibCompressor(conf); assertNotNull("zlibCompressor is null!", zlibCompressor); assertTrue("ZlibFactory returned unexpected deflator", useNative ? zlibCompressor instanceof ZlibCompressor : zlibCompressor instanceof BuiltInZlibDeflater); CodecPool.returnCompressor(zlibCompressor); // Create a GZIP text file via the Compressor interface. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.getCodec(new Path("foo.gz")); assertTrue("Codec for .gz file is not GzipCodec", codec instanceof GzipCodec); final String msg = "This is the message we are going to compress."; final String tmpDir = System.getProperty("test.build.data", "/tmp/"); final String fileName = new Path(new Path(tmpDir), "testGzipCodecWrite.txt.gz").toString(); BufferedWriter w = null; Compressor gzipCompressor = CodecPool.getCompressor(codec); if (null != gzipCompressor) { // If it gives us back a Compressor, we should be able to use this // to write files we can then read back with Java's gzip tools. OutputStream os = new CompressorStream(new FileOutputStream(fileName), gzipCompressor); w = new BufferedWriter(new OutputStreamWriter(os)); w.write(msg); w.close(); CodecPool.returnCompressor(gzipCompressor); verifyGzipFile(fileName, msg); } // Create a gzip text file via codec.getOutputStream(). w = new BufferedWriter(new OutputStreamWriter( codec.createOutputStream(new FileOutputStream(fileName)))); w.write(msg); w.close(); verifyGzipFile(fileName, msg); }
public void testGzipCodecWrite(boolean useNative) throws IOException { // Create a gzipped file using a compressor from the CodecPool, // and try to read it back via the regular GZIPInputStream. // Use native libs per the parameter Configuration conf = new Configuration(); conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, useNative); if (useNative) { if (!ZlibFactory.isNativeZlibLoaded(conf)) { LOG.warn("testGzipCodecWrite skipped: native libs not loaded"); return; } } else { assertFalse("ZlibFactory is using native libs against request", ZlibFactory.isNativeZlibLoaded(conf)); } // Ensure that the CodecPool has a BuiltInZlibDeflater in it. Compressor zlibCompressor = ZlibFactory.getZlibCompressor(conf); assertNotNull("zlibCompressor is null!", zlibCompressor); assertTrue("ZlibFactory returned unexpected deflator", useNative ? zlibCompressor instanceof ZlibCompressor : zlibCompressor instanceof BuiltInZlibDeflater); CodecPool.returnCompressor(zlibCompressor); // Create a GZIP text file via the Compressor interface. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.getCodec(new Path("foo.gz")); assertTrue("Codec for .gz file is not GzipCodec", codec instanceof GzipCodec); final String msg = "This is the message we are going to compress."; final String tmpDir = System.getProperty("test.build.data", "/tmp/"); final String fileName = new Path(new Path(tmpDir), "testGzipCodecWrite.txt.gz").toString(); BufferedWriter w = null; Compressor gzipCompressor = CodecPool.getCompressor(codec); if (null != gzipCompressor) { // If it gives us back a Compressor, we should be able to use this // to write files we can then read back with Java's gzip tools. OutputStream os = new CompressorStream(new FileOutputStream(fileName), gzipCompressor); w = new BufferedWriter(new OutputStreamWriter(os)); w.write(msg); w.close(); CodecPool.returnCompressor(gzipCompressor); verifyGzipFile(fileName, msg); } // Create a gzip text file via codec.getOutputStream(). w = new BufferedWriter(new OutputStreamWriter( codec.createOutputStream(new FileOutputStream(fileName)))); w.write(msg); w.close(); verifyGzipFile(fileName, msg); }
private void testGzipCodecWrite(boolean useNative) throws IOException { // Create a gzipped file using a compressor from the CodecPool, // and try to read it back via the regular GZIPInputStream. // Use native libs per the parameter Configuration conf = new Configuration(); conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, useNative); if (useNative) { if (!ZlibFactory.isNativeZlibLoaded(conf)) { LOG.warn("testGzipCodecWrite skipped: native libs not loaded"); return; } } else { assertFalse("ZlibFactory is using native libs against request", ZlibFactory.isNativeZlibLoaded(conf)); } // Ensure that the CodecPool has a BuiltInZlibDeflater in it. Compressor zlibCompressor = ZlibFactory.getZlibCompressor(conf); assertNotNull("zlibCompressor is null!", zlibCompressor); assertTrue("ZlibFactory returned unexpected deflator", useNative ? zlibCompressor instanceof ZlibCompressor : zlibCompressor instanceof BuiltInZlibDeflater); CodecPool.returnCompressor(zlibCompressor); // Create a GZIP text file via the Compressor interface. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.getCodec(new Path("foo.gz")); assertTrue("Codec for .gz file is not GzipCodec", codec instanceof GzipCodec); final String msg = "This is the message we are going to compress."; final String fileName = new Path(GenericTestUtils.getTempPath( "testGzipCodecWrite.txt.gz")).toString(); BufferedWriter w = null; Compressor gzipCompressor = CodecPool.getCompressor(codec); if (null != gzipCompressor) { // If it gives us back a Compressor, we should be able to use this // to write files we can then read back with Java's gzip tools. OutputStream os = new CompressorStream(new FileOutputStream(fileName), gzipCompressor); w = new BufferedWriter(new OutputStreamWriter(os)); w.write(msg); w.close(); CodecPool.returnCompressor(gzipCompressor); verifyGzipFile(fileName, msg); } // Create a gzip text file via codec.getOutputStream(). w = new BufferedWriter(new OutputStreamWriter( codec.createOutputStream(new FileOutputStream(fileName)))); w.write(msg); w.close(); verifyGzipFile(fileName, msg); }
public void testGzipCodecWrite() throws IOException { // Create a gzipped file using a compressor from the CodecPool, // and try to read it back via the regular GZIPInputStream. // Don't use native libs for this test. Configuration conf = new Configuration(); conf.setBoolean("hadoop.native.lib", false); assertFalse("ZlibFactory is using native libs against request", ZlibFactory.isNativeZlibLoaded(conf)); // Ensure that the CodecPool has a BuiltInZlibDeflater in it. Compressor zlibCompressor = ZlibFactory.getZlibCompressor(conf); assertNotNull("zlibCompressor is null!", zlibCompressor); assertTrue("ZlibFactory returned unexpected deflator", zlibCompressor instanceof BuiltInZlibDeflater); CodecPool.returnCompressor(zlibCompressor); // Create a GZIP text file via the Compressor interface. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.getCodec(new Path("foo.gz")); assertTrue("Codec for .gz file is not GzipCodec", codec instanceof GzipCodec); final String msg = "This is the message we are going to compress."; final String tmpDir = System.getProperty("test.build.data", "/tmp/"); final String fileName = new Path(new Path(tmpDir), "testGzipCodecWrite.txt.gz").toString(); BufferedWriter w = null; Compressor gzipCompressor = CodecPool.getCompressor(codec); if (null != gzipCompressor) { // If it gives us back a Compressor, we should be able to use this // to write files we can then read back with Java's gzip tools. OutputStream os = new CompressorStream(new FileOutputStream(fileName), gzipCompressor); w = new BufferedWriter(new OutputStreamWriter(os)); w.write(msg); w.close(); CodecPool.returnCompressor(gzipCompressor); verifyGzipFile(fileName, msg); } // Create a gzip text file via codec.getOutputStream(). w = new BufferedWriter(new OutputStreamWriter( codec.createOutputStream(new FileOutputStream(fileName)))); w.write(msg); w.close(); verifyGzipFile(fileName, msg); }
