Java 类org.apache.hadoop.io.compress.bzip2.Bzip2Compressor 实例源码
项目:hadoop-oss
文件:TestBzip2CompressorDecompressor.java
@Test
public void testCompressDecompress() {
byte[] rawData = null;
int rawDataSize = 0;
rawDataSize = 1024 * 64;
rawData = generate(rawDataSize);
try {
Bzip2Compressor compressor = new Bzip2Compressor();
Bzip2Decompressor decompressor = new Bzip2Decompressor();
assertFalse("testBzip2CompressDecompress finished error",
compressor.finished());
compressor.setInput(rawData, 0, rawData.length);
assertTrue("testBzip2CompressDecompress getBytesRead before error",
compressor.getBytesRead() == 0);
compressor.finish();
byte[] compressedResult = new byte[rawDataSize];
int cSize = compressor.compress(compressedResult, 0, rawDataSize);
assertTrue("testBzip2CompressDecompress getBytesRead after error",
compressor.getBytesRead() == rawDataSize);
assertTrue(
"testBzip2CompressDecompress compressed size no less than original size",
cSize < rawDataSize);
decompressor.setInput(compressedResult, 0, cSize);
byte[] decompressedBytes = new byte[rawDataSize];
decompressor.decompress(decompressedBytes, 0, decompressedBytes.length);
assertArrayEquals("testBzip2CompressDecompress arrays not equals ",
rawData, decompressedBytes);
compressor.reset();
decompressor.reset();
} catch (IOException ex) {
fail("testBzip2CompressDecompress ex !!!" + ex);
}
}
项目:hadoop
文件:Bzip2Factory.java
/**
* Check if native-bzip2 code is loaded & initialized correctly and
* can be loaded for this job.
*
* @param conf configuration
* @return <code>true</code> if native-bzip2 is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
public static boolean isNativeBzip2Loaded(Configuration conf) {
String libname = conf.get("io.compression.codec.bzip2.library",
"system-native");
if (!bzip2LibraryName.equals(libname)) {
nativeBzip2Loaded = false;
bzip2LibraryName = libname;
if (libname.equals("java-builtin")) {
LOG.info("Using pure-Java version of bzip2 library");
} else if (conf.getBoolean(
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) &&
NativeCodeLoader.isNativeCodeLoaded()) {
try {
// Initialize the native library.
Bzip2Compressor.initSymbols(libname);
Bzip2Decompressor.initSymbols(libname);
nativeBzip2Loaded = true;
LOG.info("Successfully loaded & initialized native-bzip2 library " +
libname);
} catch (Throwable t) {
LOG.warn("Failed to load/initialize native-bzip2 library " +
libname + ", will use pure-Java version");
}
}
}
return nativeBzip2Loaded;
}
项目:hadoop
文件:Bzip2Factory.java
public static String getLibraryName(Configuration conf) {
if (isNativeBzip2Loaded(conf)) {
return Bzip2Compressor.getLibraryName();
} else {
return bzip2LibraryName;
}
}
项目:aliyun-oss-hadoop-fs
文件:TestBzip2CompressorDecompressor.java
@Test
public void testCompressDecompress() {
byte[] rawData = null;
int rawDataSize = 0;
rawDataSize = 1024 * 64;
rawData = generate(rawDataSize);
try {
Bzip2Compressor compressor = new Bzip2Compressor();
Bzip2Decompressor decompressor = new Bzip2Decompressor();
assertFalse("testBzip2CompressDecompress finished error",
compressor.finished());
compressor.setInput(rawData, 0, rawData.length);
assertTrue("testBzip2CompressDecompress getBytesRead before error",
compressor.getBytesRead() == 0);
compressor.finish();
byte[] compressedResult = new byte[rawDataSize];
int cSize = compressor.compress(compressedResult, 0, rawDataSize);
assertTrue("testBzip2CompressDecompress getBytesRead after error",
compressor.getBytesRead() == rawDataSize);
assertTrue(
"testBzip2CompressDecompress compressed size no less than original size",
cSize < rawDataSize);
decompressor.setInput(compressedResult, 0, cSize);
byte[] decompressedBytes = new byte[rawDataSize];
decompressor.decompress(decompressedBytes, 0, decompressedBytes.length);
assertArrayEquals("testBzip2CompressDecompress arrays not equals ",
rawData, decompressedBytes);
compressor.reset();
decompressor.reset();
} catch (IOException ex) {
fail("testBzip2CompressDecompress ex !!!" + ex);
}
}
项目:big-c
文件:Bzip2Factory.java
/**
* Check if native-bzip2 code is loaded & initialized correctly and
* can be loaded for this job.
*
* @param conf configuration
* @return <code>true</code> if native-bzip2 is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
public static boolean isNativeBzip2Loaded(Configuration conf) {
String libname = conf.get("io.compression.codec.bzip2.library",
"system-native");
if (!bzip2LibraryName.equals(libname)) {
nativeBzip2Loaded = false;
bzip2LibraryName = libname;
if (libname.equals("java-builtin")) {
LOG.info("Using pure-Java version of bzip2 library");
} else if (conf.getBoolean(
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) &&
NativeCodeLoader.isNativeCodeLoaded()) {
try {
// Initialize the native library.
Bzip2Compressor.initSymbols(libname);
Bzip2Decompressor.initSymbols(libname);
nativeBzip2Loaded = true;
LOG.info("Successfully loaded & initialized native-bzip2 library " +
libname);
} catch (Throwable t) {
LOG.warn("Failed to load/initialize native-bzip2 library " +
libname + ", will use pure-Java version");
}
}
}
return nativeBzip2Loaded;
}
项目:big-c
文件:Bzip2Factory.java
public static String getLibraryName(Configuration conf) {
if (isNativeBzip2Loaded(conf)) {
return Bzip2Compressor.getLibraryName();
} else {
return bzip2LibraryName;
}
}
项目:hadoop-2.6.0-cdh5.4.3
文件:Bzip2Factory.java
/**
* Check if native-bzip2 code is loaded & initialized correctly and
* can be loaded for this job.
*
* @param conf configuration
* @return <code>true</code> if native-bzip2 is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
public static boolean isNativeBzip2Loaded(Configuration conf) {
String libname = conf.get("io.compression.codec.bzip2.library",
"system-native");
if (!bzip2LibraryName.equals(libname)) {
nativeBzip2Loaded = false;
bzip2LibraryName = libname;
if (libname.equals("java-builtin")) {
LOG.info("Using pure-Java version of bzip2 library");
} else if (conf.getBoolean(
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) &&
NativeCodeLoader.isNativeCodeLoaded()) {
try {
// Initialize the native library.
Bzip2Compressor.initSymbols(libname);
Bzip2Decompressor.initSymbols(libname);
nativeBzip2Loaded = true;
LOG.info("Successfully loaded & initialized native-bzip2 library " +
libname);
} catch (Throwable t) {
LOG.warn("Failed to load/initialize native-bzip2 library " +
libname + ", will use pure-Java version");
}
}
}
return nativeBzip2Loaded;
}
项目:hadoop-2.6.0-cdh5.4.3
文件:Bzip2Factory.java
public static String getLibraryName(Configuration conf) {
if (isNativeBzip2Loaded(conf)) {
return Bzip2Compressor.getLibraryName();
} else {
return bzip2LibraryName;
}
}
项目:hadoop-plus
文件:Bzip2Factory.java
/**
* Check if native-bzip2 code is loaded & initialized correctly and
* can be loaded for this job.
*
* @param conf configuration
* @return <code>true</code> if native-bzip2 is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
public static boolean isNativeBzip2Loaded(Configuration conf) {
String libname = conf.get("io.compression.codec.bzip2.library",
"system-native");
if (!bzip2LibraryName.equals(libname)) {
nativeBzip2Loaded = false;
bzip2LibraryName = libname;
if (libname.equals("java-builtin")) {
LOG.info("Using pure-Java version of bzip2 library");
} else if (conf.getBoolean(
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) &&
NativeCodeLoader.isNativeCodeLoaded()) {
try {
// Initialize the native library.
Bzip2Compressor.initSymbols(libname);
Bzip2Decompressor.initSymbols(libname);
nativeBzip2Loaded = true;
LOG.info("Successfully loaded & initialized native-bzip2 library " +
libname);
} catch (Throwable t) {
LOG.warn("Failed to load/initialize native-bzip2 library " +
libname + ", will use pure-Java version");
}
}
}
return nativeBzip2Loaded;
}
项目:hadoop-plus
文件:Bzip2Factory.java
public static String getLibraryName(Configuration conf) {
if (isNativeBzip2Loaded(conf)) {
return Bzip2Compressor.getLibraryName();
} else {
return bzip2LibraryName;
}
}
项目:hops
文件:Bzip2Factory.java
/**
* Check if native-bzip2 code is loaded & initialized correctly and
* can be loaded for this job.
*
* @param conf configuration
* @return <code>true</code> if native-bzip2 is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
public static synchronized boolean isNativeBzip2Loaded(Configuration conf) {
String libname = conf.get("io.compression.codec.bzip2.library",
"system-native");
if (!bzip2LibraryName.equals(libname)) {
nativeBzip2Loaded = false;
bzip2LibraryName = libname;
if (libname.equals("java-builtin")) {
LOG.info("Using pure-Java version of bzip2 library");
} else if (conf.getBoolean(
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) &&
NativeCodeLoader.isNativeCodeLoaded()) {
try {
// Initialize the native library.
Bzip2Compressor.initSymbols(libname);
Bzip2Decompressor.initSymbols(libname);
nativeBzip2Loaded = true;
LOG.info("Successfully loaded & initialized native-bzip2 library " +
libname);
} catch (Throwable t) {
LOG.warn("Failed to load/initialize native-bzip2 library " +
libname + ", will use pure-Java version");
}
}
}
return nativeBzip2Loaded;
}
项目:hops
文件:Bzip2Factory.java
public static String getLibraryName(Configuration conf) {
if (isNativeBzip2Loaded(conf)) {
return Bzip2Compressor.getLibraryName();
} else {
return bzip2LibraryName;
}
}
项目:hops
文件:TestBzip2CompressorDecompressor.java
@Test
public void testCompressDecompress() {
byte[] rawData = null;
int rawDataSize = 0;
rawDataSize = 1024 * 64;
rawData = generate(rawDataSize);
try {
Bzip2Compressor compressor = new Bzip2Compressor();
Bzip2Decompressor decompressor = new Bzip2Decompressor();
assertFalse("testBzip2CompressDecompress finished error",
compressor.finished());
compressor.setInput(rawData, 0, rawData.length);
assertTrue("testBzip2CompressDecompress getBytesRead before error",
compressor.getBytesRead() == 0);
compressor.finish();
byte[] compressedResult = new byte[rawDataSize];
int cSize = compressor.compress(compressedResult, 0, rawDataSize);
assertTrue("testBzip2CompressDecompress getBytesRead after error",
compressor.getBytesRead() == rawDataSize);
assertTrue(
"testBzip2CompressDecompress compressed size no less than original size",
cSize < rawDataSize);
decompressor.setInput(compressedResult, 0, cSize);
byte[] decompressedBytes = new byte[rawDataSize];
decompressor.decompress(decompressedBytes, 0, decompressedBytes.length);
assertArrayEquals("testBzip2CompressDecompress arrays not equals ",
rawData, decompressedBytes);
compressor.reset();
decompressor.reset();
} catch (IOException ex) {
fail("testBzip2CompressDecompress ex !!!" + ex);
}
}
项目:hadoop-TCP
文件:Bzip2Factory.java
/**
* Check if native-bzip2 code is loaded & initialized correctly and
* can be loaded for this job.
*
* @param conf configuration
* @return <code>true</code> if native-bzip2 is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
public static boolean isNativeBzip2Loaded(Configuration conf) {
String libname = conf.get("io.compression.codec.bzip2.library",
"system-native");
if (!bzip2LibraryName.equals(libname)) {
nativeBzip2Loaded = false;
bzip2LibraryName = libname;
if (libname.equals("java-builtin")) {
LOG.info("Using pure-Java version of bzip2 library");
} else if (conf.getBoolean(
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) &&
NativeCodeLoader.isNativeCodeLoaded()) {
try {
// Initialize the native library.
Bzip2Compressor.initSymbols(libname);
Bzip2Decompressor.initSymbols(libname);
nativeBzip2Loaded = true;
LOG.info("Successfully loaded & initialized native-bzip2 library " +
libname);
} catch (Throwable t) {
LOG.warn("Failed to load/initialize native-bzip2 library " +
libname + ", will use pure-Java version");
}
}
}
return nativeBzip2Loaded;
}
项目:hadoop-TCP
文件:Bzip2Factory.java
public static String getLibraryName(Configuration conf) {
if (isNativeBzip2Loaded(conf)) {
return Bzip2Compressor.getLibraryName();
} else {
return bzip2LibraryName;
}
}
项目:hardfs
文件:Bzip2Factory.java
/**
* Check if native-bzip2 code is loaded & initialized correctly and
* can be loaded for this job.
*
* @param conf configuration
* @return <code>true</code> if native-bzip2 is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
public static boolean isNativeBzip2Loaded(Configuration conf) {
String libname = conf.get("io.compression.codec.bzip2.library",
"system-native");
if (!bzip2LibraryName.equals(libname)) {
nativeBzip2Loaded = false;
bzip2LibraryName = libname;
if (libname.equals("java-builtin")) {
LOG.info("Using pure-Java version of bzip2 library");
} else if (conf.getBoolean(
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) &&
NativeCodeLoader.isNativeCodeLoaded()) {
try {
// Initialize the native library.
Bzip2Compressor.initSymbols(libname);
Bzip2Decompressor.initSymbols(libname);
nativeBzip2Loaded = true;
LOG.info("Successfully loaded & initialized native-bzip2 library " +
libname);
} catch (Throwable t) {
LOG.warn("Failed to load/initialize native-bzip2 library " +
libname + ", will use pure-Java version");
}
}
}
return nativeBzip2Loaded;
}
项目:hardfs
文件:Bzip2Factory.java
public static String getLibraryName(Configuration conf) {
if (isNativeBzip2Loaded(conf)) {
return Bzip2Compressor.getLibraryName();
} else {
return bzip2LibraryName;
}
}
项目:hadoop-on-lustre2
文件:Bzip2Factory.java
/**
* Check if native-bzip2 code is loaded & initialized correctly and
* can be loaded for this job.
*
* @param conf configuration
* @return <code>true</code> if native-bzip2 is loaded & initialized
* and can be loaded for this job, else <code>false</code>
*/
public static boolean isNativeBzip2Loaded(Configuration conf) {
String libname = conf.get("io.compression.codec.bzip2.library",
"system-native");
if (!bzip2LibraryName.equals(libname)) {
nativeBzip2Loaded = false;
bzip2LibraryName = libname;
if (libname.equals("java-builtin")) {
LOG.info("Using pure-Java version of bzip2 library");
} else if (conf.getBoolean(
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) &&
NativeCodeLoader.isNativeCodeLoaded()) {
try {
// Initialize the native library.
Bzip2Compressor.initSymbols(libname);
Bzip2Decompressor.initSymbols(libname);
nativeBzip2Loaded = true;
LOG.info("Successfully loaded & initialized native-bzip2 library " +
libname);
} catch (Throwable t) {
LOG.warn("Failed to load/initialize native-bzip2 library " +
libname + ", will use pure-Java version");
}
}
}
return nativeBzip2Loaded;
}
项目:hadoop-on-lustre2
文件:Bzip2Factory.java
public static String getLibraryName(Configuration conf) {
if (isNativeBzip2Loaded(conf)) {
return Bzip2Compressor.getLibraryName();
} else {
return bzip2LibraryName;
}
}
项目:hadoop
文件:Bzip2Factory.java
public static int getBlockSize(Configuration conf) {
return conf.getInt("bzip2.compress.blocksize",
Bzip2Compressor.DEFAULT_BLOCK_SIZE);
}
项目:hadoop
文件:Bzip2Factory.java
public static int getWorkFactor(Configuration conf) {
return conf.getInt("bzip2.compress.workfactor",
Bzip2Compressor.DEFAULT_WORK_FACTOR);
}
项目:big-c
文件:Bzip2Factory.java
public static int getBlockSize(Configuration conf) {
return conf.getInt("bzip2.compress.blocksize",
Bzip2Compressor.DEFAULT_BLOCK_SIZE);
}
项目:big-c
文件:Bzip2Factory.java
public static int getWorkFactor(Configuration conf) {
return conf.getInt("bzip2.compress.workfactor",
Bzip2Compressor.DEFAULT_WORK_FACTOR);
}
项目:hadoop-2.6.0-cdh5.4.3
文件:Bzip2Factory.java
public static int getBlockSize(Configuration conf) {
return conf.getInt("bzip2.compress.blocksize",
Bzip2Compressor.DEFAULT_BLOCK_SIZE);
}
项目:hadoop-2.6.0-cdh5.4.3
文件:Bzip2Factory.java
public static int getWorkFactor(Configuration conf) {
return conf.getInt("bzip2.compress.workfactor",
Bzip2Compressor.DEFAULT_WORK_FACTOR);
}
项目:hadoop-plus
文件:Bzip2Factory.java
public static int getBlockSize(Configuration conf) {
return conf.getInt("bzip2.compress.blocksize",
Bzip2Compressor.DEFAULT_BLOCK_SIZE);
}
项目:hadoop-plus
文件:Bzip2Factory.java
public static int getWorkFactor(Configuration conf) {
return conf.getInt("bzip2.compress.workfactor",
Bzip2Compressor.DEFAULT_WORK_FACTOR);
}
项目:hops
文件:Bzip2Factory.java
public static int getBlockSize(Configuration conf) {
return conf.getInt("bzip2.compress.blocksize",
Bzip2Compressor.DEFAULT_BLOCK_SIZE);
}
项目:hops
文件:Bzip2Factory.java
public static int getWorkFactor(Configuration conf) {
return conf.getInt("bzip2.compress.workfactor",
Bzip2Compressor.DEFAULT_WORK_FACTOR);
}
项目:hadoop-TCP
文件:Bzip2Factory.java
public static int getBlockSize(Configuration conf) {
return conf.getInt("bzip2.compress.blocksize",
Bzip2Compressor.DEFAULT_BLOCK_SIZE);
}
项目:hadoop-TCP
文件:Bzip2Factory.java
public static int getWorkFactor(Configuration conf) {
return conf.getInt("bzip2.compress.workfactor",
Bzip2Compressor.DEFAULT_WORK_FACTOR);
}
项目:hardfs
文件:Bzip2Factory.java
public static int getBlockSize(Configuration conf) {
return conf.getInt("bzip2.compress.blocksize",
Bzip2Compressor.DEFAULT_BLOCK_SIZE);
}
项目:hardfs
文件:Bzip2Factory.java
public static int getWorkFactor(Configuration conf) {
return conf.getInt("bzip2.compress.workfactor",
Bzip2Compressor.DEFAULT_WORK_FACTOR);
}
项目:hadoop-on-lustre2
文件:Bzip2Factory.java
public static int getBlockSize(Configuration conf) {
return conf.getInt("bzip2.compress.blocksize",
Bzip2Compressor.DEFAULT_BLOCK_SIZE);
}
项目:hadoop-on-lustre2
文件:Bzip2Factory.java
public static int getWorkFactor(Configuration conf) {
return conf.getInt("bzip2.compress.workfactor",
Bzip2Compressor.DEFAULT_WORK_FACTOR);
}
项目:hadoop
文件:Bzip2Factory.java
/**
* Return the appropriate type of the bzip2 compressor.
*
* @param conf configuration
* @return the appropriate type of the bzip2 compressor.
*/
public static Class<? extends Compressor>
getBzip2CompressorType(Configuration conf) {
return isNativeBzip2Loaded(conf) ?
Bzip2Compressor.class : BZip2DummyCompressor.class;
}
项目:hadoop
文件:Bzip2Factory.java
/**
* Return the appropriate implementation of the bzip2 compressor.
*
* @param conf configuration
* @return the appropriate implementation of the bzip2 compressor.
*/
public static Compressor getBzip2Compressor(Configuration conf) {
return isNativeBzip2Loaded(conf)?
new Bzip2Compressor(conf) : new BZip2DummyCompressor();
}
项目:big-c
文件:Bzip2Factory.java
/**
* Return the appropriate type of the bzip2 compressor.
*
* @param conf configuration
* @return the appropriate type of the bzip2 compressor.
*/
public static Class<? extends Compressor>
getBzip2CompressorType(Configuration conf) {
return isNativeBzip2Loaded(conf) ?
Bzip2Compressor.class : BZip2DummyCompressor.class;
}
项目:big-c
文件:Bzip2Factory.java
/**
* Return the appropriate implementation of the bzip2 compressor.
*
* @param conf configuration
* @return the appropriate implementation of the bzip2 compressor.
*/
public static Compressor getBzip2Compressor(Configuration conf) {
return isNativeBzip2Loaded(conf)?
new Bzip2Compressor(conf) : new BZip2DummyCompressor();
}
项目:hadoop-2.6.0-cdh5.4.3
文件:Bzip2Factory.java
/**
* Return the appropriate type of the bzip2 compressor.
*
* @param conf configuration
* @return the appropriate type of the bzip2 compressor.
*/
public static Class<? extends Compressor>
getBzip2CompressorType(Configuration conf) {
return isNativeBzip2Loaded(conf) ?
Bzip2Compressor.class : BZip2DummyCompressor.class;
}