/** * This method checks if the given file is a Zip file containing one entry (in case of file * extension .zip). If this is the case, a reader based on a ZipInputStream for this entry is * returned. Otherwise, this method checks if the file has the extension .gz. If this applies, a * gzipped stream reader is returned. Otherwise, this method just returns a BufferedReader for * the given file (file was not zipped at all). */ public static BufferedReader getReader(File file, Charset encoding) throws IOException { // handle zip files if necessary if (file.getAbsolutePath().endsWith(".zip")) { try (ZipFile zipFile = new ZipFile(file)) { if (zipFile.size() == 0) { throw new IOException("Input of Zip file failed: the file archive does not contain any entries."); } if (zipFile.size() > 1) { throw new IOException("Input of Zip file failed: the file archive contains more than one entry."); } Enumeration<? extends ZipEntry> entries = zipFile.entries(); InputStream zipIn = zipFile.getInputStream(entries.nextElement()); return new BufferedReader(new InputStreamReader(zipIn, encoding)); } } else if (file.getAbsolutePath().endsWith(".gz")) { return new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(file)), encoding)); } else { return new BufferedReader(new InputStreamReader(new FileInputStream(file), encoding)); } }
static void init() { try { glossary = new Vector(); URL url = URLFactory.url( DSDP.ROOT+"fauna/glossary.gz"); BufferedReader in = new BufferedReader( new InputStreamReader( new GZIPInputStream( url.openStream() ))); String s; while( (s=in.readLine())!=null ) glossary.add(s.getBytes()); glossary.trimToSize(); } catch(IOException e) { e.printStackTrace(); } }
/** * Checks the InputStream if it contains GZIP compressed data * * @param inputStream InputStream to be checked * @return true or false if the stream contains GZIP compressed data * @throws java.io.IOException if read from inputStream fails */ public static boolean isInputStreamGZIPCompressed(final PushbackInputStream inputStream) throws IOException { if (inputStream == null) return false; byte[] signature = new byte[2]; int count = 0; try { while (count < 2) { int readCount = inputStream.read(signature, count, 2 - count); if (readCount < 0) return false; count = count + readCount; } } finally { inputStream.unread(signature, 0, count); } int streamHeader = ((int) signature[0] & 0xff) | ((signature[1] << 8) & 0xff00); return GZIPInputStream.GZIP_MAGIC == streamHeader; }
static public InputStream createInput(File file) { if (file == null) { throw new IllegalArgumentException("File passed to createInput() was null"); } try { InputStream input = new FileInputStream(file); if (file.getName().toLowerCase().endsWith(".gz")) { return new GZIPInputStream(input); } return input; } catch (IOException e) { System.err.println("Could not createInput() for " + file); e.printStackTrace(); return null; } }
public static void read(File file, Consumer<File> handle) throws IOException { Path temp; if (SystemUtils.IS_OS_LINUX) { temp = Files.createTempFile(SHARED_MEMORY, TEMP_PREFIX, null); } else { temp = Files.createTempFile(TEMP_PREFIX, null); } try ( InputStream is = new FileInputStream(file); GZIPInputStream gis = new GZIPInputStream(is); OutputStream os = Files.newOutputStream(temp); ) { byte[] buffer = new byte[1024]; int length; while ((length = gis.read(buffer, 0, 1024)) != -1) { os.write(buffer, 0, length); } handle.accept(temp.toFile()); } finally { Files.delete(temp); } }
private static GameFile getGameFileFromFile(String file) throws JAXBException, IOException { final JAXBContext jaxbContext = JAXBContext.newInstance(GameFile.class); final Unmarshaller um = jaxbContext.createUnmarshaller(); try (InputStream inputStream = FileUtilities.getGameResource(file)) { // try to get compressed game file final GZIPInputStream zipStream = new GZIPInputStream(inputStream); return (GameFile) um.unmarshal(zipStream); } catch (final ZipException e) { // if it fails to load the compressed file, get it from plain XML InputStream stream = null; stream = FileUtilities.getGameResource(file); if (stream == null) { return null; } return (GameFile) um.unmarshal(stream); } }
private void loadHexFiles() throws FileNotFoundException, IOException { FilenameFilter filter = new UniverseFilter("hex"); File directory = new File("data"); File[] files = directory.listFiles(filter); for (File file : files) { FileInputStream fos = new FileInputStream(file); // Save to file GZIPInputStream gzos = new GZIPInputStream(fos); DataInputStream data = new DataInputStream(gzos); int count = data.readInt(); List<Coord> newMap = new LinkedList<Coord>(); for (int i = 0; i < count; i++) { newMap.add(new Coord(data.readInt(), data.readInt(), data.readInt())); data.readInt(); } data.close(); // Close the stream. _log.info(newMap.size() + " map vertices loaded from file " + file.getName()); _coordList.addAll(newMap); } }
@SuppressWarnings(value = {"unchecked"}) private void loadBinFiles() throws FileNotFoundException, IOException, ClassNotFoundException { FilenameFilter filter = new UniverseFilter("bin"); File directory = new File("data"); File[] files = directory.listFiles(filter); for (File file : files) { //Create necessary input streams FileInputStream fis = new FileInputStream(file); // Read from file GZIPInputStream gzis = new GZIPInputStream(fis); // Uncompress ObjectInputStream in = new ObjectInputStream(gzis); // Read objects // Read in an object. It should be a vector of scribbles TreeSet<Position> temp = (TreeSet<Position>) in.readObject(); _log.info(temp.size() + " map vertices loaded from file " + file.getName()); in.close(); // Close the stream. for (Position p : temp) { _coordList.add(new Coord(p._x, p._y, p._z)); } } }
public StatArchiveFile(StatArchiveReader reader, File archiveName, boolean dump, ValueFilter[] filters) throws IOException { this.reader = reader; this.archiveName = archiveName; this.dump = dump; this.compressed = archiveName.getPath().endsWith(".gz"); this.is = new FileInputStream(this.archiveName); if (this.compressed) { this.dataIn = new DataInputStream( new BufferedInputStream(new GZIPInputStream(this.is, BUFFER_SIZE), BUFFER_SIZE)); } else { this.dataIn = new DataInputStream(new BufferedInputStream(this.is, BUFFER_SIZE)); } this.updateOK = this.dataIn.markSupported(); this.filters = createFilters(filters); }
public String readCompressedFile(String fileName) { try { GZIPInputStream gis = new GZIPInputStream(new FileInputStream(fileName)); ByteArrayOutputStream fos = new ByteArrayOutputStream(); byte[] buffer = new byte[1024]; int len; while ((len = gis.read(buffer)) != -1) { fos.write(buffer, 0, len); } fos.close(); gis.close(); return new String(fos.toByteArray()); } catch (IOException ex) { System.out.println("Invalid input file!"); return null; } }
public static String gzipUncompressToString(byte[] b) { if (b == null || b.length == 0) { return null; } ByteArrayOutputStream out = new ByteArrayOutputStream(); ByteArrayInputStream in = new ByteArrayInputStream(b); try { GZIPInputStream gunzip = new GZIPInputStream(in); byte[] buffer = new byte[256]; int n; while ((n = gunzip.read(buffer)) >= 0) { out.write(buffer, 0, n); } } catch (IOException e) { e.printStackTrace(); } return out.toString(); }
static void init() { try { glossary = new Vector(); URL url = URLFactory.url( DSDP.ROOT+"authors.list.gz"); BufferedReader in = new BufferedReader( new InputStreamReader( new GZIPInputStream( url.openStream() ))); String s = in.readLine(); while( (s=in.readLine())!=null ) glossary.add(s.getBytes()); glossary.trimToSize(); } catch(IOException e) { } }
/** * Creates a BufferedReader for the given input path Can handle both gzip * and plain text files * * @param file * @return * @throws IOException */ public static BufferedReader getReader(File file) throws IOException { if (!file.exists()) { throw new IOException("The file '" + file + "' does not exist"); } BufferedReader in = null; if (file.getPath().endsWith(".gz")) { FileInputStream fin = new FileInputStream(file); GZIPInputStream gzis = new GZIPInputStream(fin); in = new BufferedReader(new InputStreamReader(gzis)); LOG.debug("Reading in the zipped contents of '" + file.getName() + "'"); } else { in = new BufferedReader(new FileReader(file)); LOG.debug("Reading in the contents of '" + file.getName() + "'"); } return (in); }
private void download(String urlString, LocalDate localDate) throws IOException { URL url = new URL(urlString); try (InputStream is = url.openStream(); GZIPInputStream gzis = new GZIPInputStream(is); InputStreamReader isr = new InputStreamReader(gzis, StandardCharsets.UTF_8); BufferedReader br = new BufferedReader(isr)) { List<InfoSchema> infos = new ArrayList<>(); String line; while ((line = br.readLine()) != null) { if (!line.isEmpty()) { InfoSchema info = parseLineToInfo(line, localDate); if (info != null) { infos.add(info); } } } this.databaseService.saveInfos(infos); } }
@Test public void testGzipDurability() throws Exception { Context context = new Context(); HDFSCompressedDataStream writer = new HDFSCompressedDataStream(); writer.configure(context); writer.open(fileURI, factory.getCodec(new Path(fileURI)), SequenceFile.CompressionType.BLOCK); String[] bodies = { "yarf!" }; writeBodies(writer, bodies); byte[] buf = new byte[256]; GZIPInputStream cmpIn = new GZIPInputStream(new FileInputStream(file)); int len = cmpIn.read(buf); String result = new String(buf, 0, len, Charsets.UTF_8); result = result.trim(); // BodyTextEventSerializer adds a newline Assert.assertEquals("input and output must match", bodies[0], result); }
/** * Decompresses a GZip'd byte array. * * @param compressedBytes * The compressed byte array. * * @return * The decompressed byte array. * * @throws IOException * If an I/O error has occurred. */ private static byte[] decompress(final byte[] compressedBytes) throws IOException { final ByteArrayInputStream byteInputStream = new ByteArrayInputStream(compressedBytes); final ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(compressedBytes.length); final GZIPInputStream gzipInputStream = new GZIPInputStream(byteInputStream); byte[] buffer = new byte[1024]; while (gzipInputStream.available() > 0) { final int count = gzipInputStream.read(buffer, 0, 1024); if (count > 0) { byteOutputStream.write(buffer, 0, count); } } gzipInputStream.close(); byteOutputStream.close(); byteInputStream.close(); return byteOutputStream.toByteArray(); }
public static void main(String[] args) throws IOException{ BufferedReader in=new BufferedReader( new FileReader("./src/main/java/io/source/data1.txt") ); BufferedOutputStream out=new BufferedOutputStream(new GZIPOutputStream( new FileOutputStream("./src/main/java/io/source/data1.gz") )); System.out.println("Writing file"); int c; while ((c=in.read())!=-1){ out.write(c); } in.close(); out.close(); System.out.println("Reading file"); BufferedReader in2=new BufferedReader( new InputStreamReader(new GZIPInputStream( new FileInputStream("./src/main/java/io/source/data1.gz")))); String s; while ((s=in2.readLine())!=null){ System.out.println(s); } }
public static Bundle deserializeBundle(final String base64) { Bundle bundle; final Parcel parcel = Parcel.obtain(); try { final ByteArrayOutputStream byteBuffer = new ByteArrayOutputStream(); final byte[] buffer = new byte[1024]; final GZIPInputStream zis = new GZIPInputStream(new ByteArrayInputStream(Base64.decode(base64, 0))); int len; while ((len = zis.read(buffer)) != -1) { byteBuffer.write(buffer, 0, len); } zis.close(); parcel.unmarshall(byteBuffer.toByteArray(), 0, byteBuffer.size()); parcel.setDataPosition(0); bundle = parcel.readBundle(); } catch (IOException e) { e.printStackTrace(); bundle = null; } finally { parcel.recycle(); } return bundle; }
private void initContentStream(InputStream transferStream) throws IOException { responseTransferIn = transferStream; if (transparentGzip && responseHeaders.isContentEncodingGzip()) { // If the response was transparently gzipped, remove the gzip header field // so clients don't double decompress. http://b/3009828 // // Also remove the Content-Length in this case because it contains the // length 528 of the gzipped response. This isn't terribly useful and is // dangerous because 529 clients can query the content length, but not // the content encoding. responseHeaders.stripContentEncoding(); responseHeaders.stripContentLength(); responseBodyIn = new GZIPInputStream(transferStream); } else { responseBodyIn = transferStream; } }
@Test public void testToBlob_withCompression() throws IOException { Blob blob = testTarStreamBuilder.toBlob(); // Writes the BLOB and captures the output. ByteArrayOutputStream tarByteOutputStream = new ByteArrayOutputStream(); OutputStream compressorStream = new GZIPOutputStream(tarByteOutputStream); blob.writeTo(compressorStream); // Rearrange the output into input for verification. ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(tarByteOutputStream.toByteArray()); InputStream tarByteInputStream = new GZIPInputStream(byteArrayInputStream); TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(tarByteInputStream); verifyTarArchive(tarArchiveInputStream); }
public static void main(String[] args) throws Throwable { // Download Semantic Web Dog Food dataset about papers and create HDT String url = "http://gaia.infor.uva.es/hdt/swdf-2012-11-28.hdt.gz"; InputStream in = new BufferedInputStream(new GZIPInputStream(new URL(url).openStream())); try(HDT hdt = HDTManager.loadIndexedHDT(in)){ in.close(); // Create a Gremlin Graph try(HDTGraph hdtgraph = new HDTGraph(hdt)){ // Find Mario's coauthors in SWDF dataset hdtgraph.traversal().V("http://data.semanticweb.org/person/mario-arias-gallego") .out("http://xmlns.com/foaf/0.1/made") .in("http://xmlns.com/foaf/0.1/made") .sideEffect( e-> System.out.println(e) ) .iterate(); } } System.exit(0); }
public static byte[] unzipBytes(byte[] bytes) { if (bytes == null || bytes.length == 0) { return bytes; } try ( ByteArrayOutputStream bos = new ByteArrayOutputStream(); ByteArrayInputStream bis = new ByteArrayInputStream(bytes); GZIPInputStream gunzip = new GZIPInputStream(bis); ) { byte[] buffer = new byte[256]; int n; while ((n = gunzip.read(buffer)) >= 0) { bos.write(buffer, 0, n); } bis.close(); gunzip.close(); bos.close(); return bos.toByteArray(); } catch (IOException e) { LOGGER.error("Can not unzip bytes, please check."); } return bytes; }
@Override protected Void call() throws Exception { Platform.runLater(this.observableList::clear); SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); SAXParser saxParser = saxParserFactory.newSAXParser(); try (ZipFile zipFile = new ZipFile(this.selectedFile)) { int workMax = zipFile.size(); long workDone = 0; for (Enumeration<? extends ZipEntry> e = zipFile.entries(); e.hasMoreElements();) { ZipEntry zipEntry = e.nextElement(); int position = zipEntry.getName().charAt(zipEntry.getName().length() - 7) - '0'; SyantenAnalyzer analyzer = new SyantenAnalyzer(position); ParseHandler parseHandler = new ParseHandler(analyzer); try (InputStream is = zipFile.getInputStream(zipEntry); GZIPInputStream gzis = new GZIPInputStream(is)) { saxParser.parse(gzis, parseHandler); } ArrayList<MahjongScene> scenes = analyzer.getOriScenes(); workDone++; Platform.runLater(() -> observableList.addAll(scenes)); updateMessage(workDone + "/" + workMax); updateProgress(workDone, workMax); } } return null; }
/** * Get the original inventory report from S3, unzip it, and transfer it into a String format. * @return inventReport String * @throws IOException when getting object from S3 fails * or the checksum of the inventory report and the checksum specified in the manifest file not match */ public String getInventoryReportToString() throws IOException { String inventReportKey = locator.getKey(); String bucketName = inventoryManifest.getSourceBucket(); try (S3Object s3InventoryReport = s3Client.getObject( new GetObjectRequest(bucketName, inventReportKey))) { InputStream objectData = s3InventoryReport.getObjectContent(); byte[] zippedData = IOUtils.toByteArray(objectData); String actualChecksum = DigestUtils.md5Hex(zippedData); String expectedChecksum = locator.getMD5checksum(); if (!actualChecksum.equals(expectedChecksum)) { throw new ChecksumMismatchException (expectedChecksum, actualChecksum); } return IOUtils.toString(new GZIPInputStream(new ByteArrayInputStream(zippedData))); } }
public static String decompressGzip (final byte[] compressed) throws IOException { final StringBuilder outStr = new StringBuilder(); if ((compressed == null) || (compressed.length == 0)) { return ""; } if (isCompressed(compressed)) { final GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(compressed)); final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(gis, "UTF-8")); String line; while ((line = bufferedReader.readLine()) != null) { outStr.append(line); } } else { outStr.append(compressed); } return outStr.toString(); }
/** * Uncompresses a GZIP file. * * @param bytes The compressed bytes. * @return The uncompressed bytes. * @throws IOException if an I/O error occurs. */ public static byte[] gunzip(byte[] bytes) throws IOException { /* create the streams */ InputStream is = new GZIPInputStream(new ByteArrayInputStream(bytes)); try { ByteArrayOutputStream os = new ByteArrayOutputStream(); try { /* copy data between the streams */ byte[] buf = new byte[4096]; int len = 0; while ((len = is.read(buf, 0, buf.length)) != -1) { os.write(buf, 0, len); } } finally { os.close(); } /* return the uncompressed bytes */ return os.toByteArray(); } finally { is.close(); } }
public static byte[] readCompressedByteArray(DataInput in) throws IOException { int length = in.readInt(); if (length == -1) return null; byte[] buffer = new byte[length]; in.readFully(buffer); // could/should use readFully(buffer,0,length)? GZIPInputStream gzi = new GZIPInputStream(new ByteArrayInputStream(buffer, 0, buffer.length)); byte[] outbuf = new byte[length]; ByteArrayOutputStream bos = new ByteArrayOutputStream(); int len; while((len=gzi.read(outbuf, 0, outbuf.length)) != -1){ bos.write(outbuf, 0, len); } byte[] decompressed = bos.toByteArray(); bos.close(); gzi.close(); return decompressed; }
public InputStream b() { if (d() < 400) { try { InputStream inputStream = a().getInputStream(); } catch (IOException e) { throw new RestException(e); } } inputStream = a().getErrorStream(); if (inputStream == null) { try { inputStream = a().getInputStream(); } catch (IOException e2) { throw new RestException(e2); } } if (!this.g || !AsyncHttpClient.ENCODING_GZIP.equals(c())) { return inputStream; } try { return new GZIPInputStream(inputStream); } catch (IOException e22) { throw new RestException(e22); } }
/** * 解压缩字符串 * * @param str * @return String * @throws IOException */ public static String uncompress(String str) throws IOException { if (str == null || str.length() == 0) { return str; } ByteArrayOutputStream out = new ByteArrayOutputStream(); ByteArrayInputStream in = new ByteArrayInputStream( str.getBytes("ISO-8859-1")); GZIPInputStream gunzip = new GZIPInputStream(in); byte[] buffer = new byte[256]; int n; while ((n = gunzip.read(buffer)) >= 0) { out.write(buffer, 0, n); } return out.toString("UTF-8"); }
/** * Load the gzipped compound from the inputstream. */ public static NBTTagCompound readCompressed(InputStream is) throws IOException { DataInputStream datainputstream = new DataInputStream(new BufferedInputStream(new GZIPInputStream(is))); NBTTagCompound nbttagcompound; try { nbttagcompound = read(datainputstream, NBTSizeTracker.INFINITE); } finally { datainputstream.close(); } return nbttagcompound; }
@Override public byte[] decompress( final byte[] data , final int start , final int length ) throws IOException{ ByteBuffer wrapBuffer = ByteBuffer.wrap( data , start , length ); int dataLength = wrapBuffer.getInt(); ByteArrayInputStream bIn = new ByteArrayInputStream( data , start + Integer.BYTES , length ); GZIPInputStream in = new GZIPInputStream( bIn , 1024 * 256 ); byte[] retVal = new byte[dataLength]; InputStreamUtils.read( in , retVal , 0 , dataLength ); return retVal; }
private static InputStream maybeDecompress(InputStream input) throws IOException { // Due to a bug, Jira sometimes returns double-compressed responses. See JRA-37608 BufferedInputStream buffered = new BufferedInputStream(input, 2); buffered.mark(2); int[] buf = new int[2]; buf[0] = buffered.read(); buf[1] = buffered.read(); buffered.reset(); int header = (buf[1] << 8) | buf[0]; if (header == GZIPInputStream.GZIP_MAGIC) { return new GZIPInputStream(buffered); } else { return buffered; } }
/** * レスポンスボディのストリームを受け取る. * @param res Responseオブジェクト * @return ストリーム * @throws IOException IO例外 */ protected final InputStream getResponseBodyInputStream(final HttpResponse res) throws IOException { // GZip 圧縮されていたら解凍する。 Header[] contentEncodingHeaders = res.getHeaders("Content-Encoding"); if (contentEncodingHeaders.length > 0 && "gzip".equalsIgnoreCase(contentEncodingHeaders[0].getValue())) { return new GZIPInputStream(res.getEntity().getContent()); } else { return res.getEntity().getContent(); } }
public InputStream read() { InputStream is = resource.read(); try { return new GZIPInputStream(is); } catch (Exception e) { IOUtils.closeQuietly(is); throw ResourceExceptions.readFailed(resource.getDisplayName(), e); } }
@SuppressWarnings("unchecked") public void read(InputStream in) throws IOException { try { ObjectInputStream oin = new ObjectInputStream(new GZIPInputStream(in)); m_superclasses = (HashMap<ClassEntry,ClassEntry>)oin.readObject(); m_fieldEntries = (HashMultimap<ClassEntry,FieldEntry>)oin.readObject(); m_behaviorEntries = (HashMultimap<ClassEntry,BehaviorEntry>)oin.readObject(); } catch (ClassNotFoundException ex) { throw new Error(ex); } }
public static InputStream getContent(HttpEntity entity) throws IOException { Header encodingHeader = entity.getContentEncoding(); String encoding = encodingHeader == null ? null : encodingHeader.getValue(); if ("gzip".equalsIgnoreCase(encoding)) { return new GZIPInputStream(entity.getContent()); } if ("deflate".equalsIgnoreCase(encoding)) { return new InflaterInputStream(entity.getContent()); } return entity.getContent(); }
/** * Expands a file compressed using {@code gzip}. * * @param fs the {@code FileSystem} corresponding to the given file. * @param in the path to the compressed file. * @param out the path to the uncompressed output. * @throws Exception if there was an error during the operation. */ private void expandGzippedTrace(FileSystem fs, Path in, Path out) throws Exception { byte[] buff = new byte[4096]; GZIPInputStream gis = new GZIPInputStream(fs.open(in)); FSDataOutputStream fsdOs = fs.create(out); int numRead; while ((numRead = gis.read(buff, 0, buff.length)) != -1) { fsdOs.write(buff, 0, numRead); } gis.close(); fsdOs.close(); }
/** * Determines if a byte array is compressed. The java.util.zip GZip * implementaiton does not expose the GZip header so it is difficult to * determine if a string is compressed. * * @return true if the array is compressed or false otherwise */ public boolean isCompressed() { if ((bytes == null) || (bytes.length < 2)) { return false; } else { return ((bytes[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (bytes[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8))); } }