Java 类org.apache.commons.io.input.BOMInputStream 实例源码

@Override
public List<String> readLines(URL url) throws IOException {

    ByteArrayOutputStream out = new ByteArrayOutputStream();

    try (InputStream in = new BOMInputStream(url.openStream());) {

        int tmp = -1;
        while ((tmp = in.read()) != -1) {
            out.write(tmp);
        }
    }

    String string = out.toString();
    LOGGER.debug(string);
    XMLDiffFormatter xmlFormatter = new XMLDiffFormatter();
    String format = xmlFormatter.format(string);

    return Stream.of(format.split("\n")).collect(Collectors.toList());
}

private BufferedReader getBufferedReader() throws IOException {
    String extension;
    Charset charset;
    if (FileUtil.isUrl(uri)) {
        extension = FileUtil.getUrlExtension(uri).toLowerCase(PMS.getLocale());
    } else {
        extension = FileUtil.getExtension(uri).toLowerCase(PMS.getLocale());
    }
    if (extension != null && (extension.equals("m3u8") || extension.equals(".cue"))) {
        charset = StandardCharsets.UTF_8;
    } else {
        charset = StandardCharsets.ISO_8859_1;
    }
    if (FileUtil.isUrl(uri)) {
        return new BufferedReader(new InputStreamReader(new BOMInputStream(new URL(uri).openStream()), charset));
    } else {
        File playlistfile = new File(uri);
        if (playlistfile.length() < 10000000) {
            return new BufferedReader(new InputStreamReader(new BOMInputStream(new FileInputStream(playlistfile)), charset));
        }
    }
    return null;
}

/**
 * Get content from {@link java.nio.file.Path} using UTF8
 *
 * @param path
 * @return
 * @throws CommandException
 */
public String getFileContent(Path path) throws CommandException {
    try {
        File file = path.toFile();
        BOMInputStream inputStream = new BOMInputStream(FileUtils.openInputStream(file), false, boms);
        String fileContent;
        if (inputStream.hasBOM()) {
            fileContent = IOUtils.toString(inputStream, inputStream.getBOMCharsetName());
        } else {
            fileContent = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
        }
        return fileContent;
    } catch (IOException e) {
        throw new CommandException("Cannot get file content for path: " + path.toString(), e);
    }
}

@Test
public void test() throws IOException {
    ApacheCommonsUtils acu = new ApacheCommonsUtils();

    FileInputStream fis = new FileInputStream(new File("src/test/resources/lotl_utf-8-sansbom.xml"));
    FileInputStream fisBom = new FileInputStream(new File("src/test/resources/lotl_utf-8.xml"));

    assertNotEquals(acu.toBase64(acu.toByteArray(fis)), acu.toBase64(acu.toByteArray(fisBom)));

    fis = new FileInputStream(new File("src/test/resources/lotl_utf-8-sansbom.xml"));
    fisBom = new FileInputStream(new File("src/test/resources/lotl_utf-8.xml"));

    BOMInputStream bomIS = new BOMInputStream(fis);
    BOMInputStream bomISSkipped = new BOMInputStream(fisBom);

    assertEquals(acu.toBase64(acu.toByteArray(bomIS)), acu.toBase64(acu.toByteArray(bomISSkipped)));
}

/**
 * Process a text-based Sitemap. Text sitemaps only list URLs but no
 * priorities, last mods, etc.
 *
 * @param sitemapUrl
 *            URL to sitemap file
 * @param stream
 *            content stream
 * @return The site map
 * @throws IOException
 *             if there is an error reading in the site map content
 */
protected SiteMap processText(URL sitemapUrl, InputStream stream) throws IOException {
    LOG.debug("Processing textual Sitemap");

    SiteMap textSiteMap = new SiteMap(sitemapUrl);
    textSiteMap.setType(SitemapType.TEXT);

    BOMInputStream bomIs = new BOMInputStream(stream);
    @SuppressWarnings("resource")
    BufferedReader reader = new BufferedReader(new InputStreamReader(bomIs, UTF_8));

    String line;
    int i = 1;
    while ((line = reader.readLine()) != null) {
        if (line.length() > 0 && i <= MAX_URLS) {
            addUrlIntoSitemap(line, textSiteMap, null, null, null, i++);
        }
    }
    textSiteMap.setProcessed(true);

    return textSiteMap;
}

/**
 * Decompress the gzipped content and process the resulting XML Sitemap.
 * 
 * @param url
 *            - URL of the gzipped content
 * @param response
 *            - Gzipped content
 * @return the site map
 * @throws UnknownFormatException
 *             if there is an error parsing the gzip
 * @throws IOException
 *             if there is an error reading in the gzip {@link java.net.URL}
 */
protected AbstractSiteMap processGzippedXML(URL url, byte[] response) throws IOException, UnknownFormatException {

    LOG.debug("Processing gzipped XML");

    InputStream is = new ByteArrayInputStream(response);

    // Remove .gz ending
    String xmlUrl = url.toString().replaceFirst("\\.gz$", "");
    LOG.debug("XML url = {}", xmlUrl);

    BOMInputStream decompressed = new BOMInputStream(new GZIPInputStream(is));
    InputSource in = new InputSource(decompressed);
    in.setSystemId(xmlUrl);
    return processXml(url, in);
}

public InputTextSubFile(SubtitleFileType inputFormat, String fileName,
        InputStream is) throws InputTextSubException, IOException {
    try {
        tto = createFormat(inputFormat).parseFile(fileName,
                new BOMInputStream(is));
    } catch (FatalParsingException ex) {
        throw new InputTextSubException(
                "Parse error returned by subtitle read library", ex);
    }
    captions = new ArrayList<InputSubtitleLine>(tto.captions.size());
    for (Caption caption : tto.captions.values()) {
        InputSubtitleLine line = new InputSubtitleLine();
        line.setContent(caption.content);
        line.setStartTime(new SubtitleFileTimeWrapper(caption.start)
                .getMSeconds());
        line.setEndTime(new SubtitleFileTimeWrapper(caption.end)
                .getMSeconds());
        captions.add(line);
    }
}

/**
 * Opens a CSV file.
 *
 * If the given file ends with "gz", then the file is decompressed before using a {@link GZIPInputStream}.
 *
 * @param importFile
 *            the csv file
 * @return a list reader
 * @throws IOException
 *             on io exception
 */
@SuppressWarnings("resource")
protected CsvListReader openCsvListReader(final File importFile) throws IOException {
    // Open file
    InputStream fileStream = new FileInputStream(importFile);

    // Check for compressed file
    if (importFile.getName().toLowerCase().endsWith(".gz")) {
        fileStream = new GZIPInputStream(fileStream);
    }

    // Guess the encoding
    final BOMInputStream inputStream = new BOMInputStream(fileStream, false, ByteOrderMark.UTF_8,
            ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE);
    final String charset;
    if (inputStream.hasBOM()) {
        charset = inputStream.getBOMCharsetName();
        log.info("BOM detected. Using {} as encoding", charset);
    } else {
        charset = getDefaultEncoding().toString();
        log.info("No BOM detected. Assuming {} as encoding", charset);
    }
    final Reader reader = new InputStreamReader(inputStream, charset);
    return new CsvListReader(reader, new CsvPreference.Builder(CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE)
            .skipComments(new CommentMatches("(//|/\\*|#|;).*")).build());
}

/**
 * Simple invocation. Load template and data from a file, run process
 * and return memory model containing results or null if there was a problem.
 * Problems/progress reporting live to given reporter
 * @param templateFile the name of the template file to use
 * @param dataFile  the name of the data file to process
 * @param report the message reporter
 * @param debug set to true to enable voluminous debug message
 * @param allowNullRows set to true to allow output even if some rows don't match
 * @throws IOException 
 */
public Model simpleConvert(String templateFile, String dataFile, ProgressMonitorReporter reporter, boolean debug, boolean allowNullRows) throws IOException {
    Template template = TemplateFactory.templateFrom(templateFile, dc);

    File dataFileF = new File(dataFile);
    String filename = dataFileF.getName();
    String filebasename = NameUtils.removeExtension(filename);
    put(ConverterProcess.FILE_NAME, filename);
    put(ConverterProcess.FILE_BASE_NAME, filebasename);
    InputStream is = new BOMInputStream( new FileInputStream(dataFileF) );

    ConverterProcess process = new ConverterProcess(dc, is);
    process.setDebug(debug);
    process.setTemplate( template );
    process.setMessageReporter( reporter );
    process.setAllowNullRows(allowNullRows);
    boolean ok = process.process();

    return ok ?  process.getModel() : null;
}

/**
 * Generate a single Jena model from several different files, output it to 
 * specified OutputStream
 * @param aggr String[] String array containing all relevant RDF files "name.extension"
 * @param out OutputStream
 * @param type an instance of ScDemoFile class
 * @throws IOException
 */
public static Model generateAggregateModel(String[] aggr, String lang) 
        throws IOException {

    Model model = ModelFactory.createDefaultModel();
    Model subModel = ModelFactory.createDefaultModel();

    for (int i=0; i<aggr.length; i++) {
        InputStream in = (JsonldJenaUtils.class).getClassLoader().getResourceAsStream(
                aggr[i]);
        BOMInputStream bIn = new BOMInputStream(in, false);

        subModel.read(bIn, null, lang);
        model = model.add(subModel);

        subModel.removeAll();
        bIn.close();
        in.close();
    }

    return model;

}

/**
 * Decompress the gzipped content and process the resulting XML Sitemap.
 * 
 * @param url - URL of the gzipped content
 * @param response - Gzipped content
 * @throws MalformedURLException
 * @throws IOException
 * @throws UnknownFormatException
 */
private AbstractSiteMap processGzip(URL url, byte[] response) throws MalformedURLException, IOException,
        UnknownFormatException {

    logger.debug("Processing gzip");

    AbstractSiteMap smi;

    InputStream is = new ByteArrayInputStream(response);

    // Remove .gz ending
    String xmlUrl = url.toString().replaceFirst("\\.gz$", "");

    logger.debug("XML url = " + xmlUrl);

    BOMInputStream decompressed = new BOMInputStream(new GZIPInputStream(is));
    InputSource in = new InputSource(decompressed);
    in.setSystemId(xmlUrl);
    smi = processXml(url, in);
    decompressed.close();
    return smi;
}

private String showByteOfMark(InputStream source) throws IOException {
  ByteOrderMark detectedBOM = new BOMInputStream(source).getBOM();
  if (detectedBOM == null) {
    return "";
  }
  String bom = detectedBOM.toString();
  FileType.logger.log(Level.INFO, "BOM: {0}", bom);
  return " w/ " + bom;
}

private static InputStream streamFile(File file) {
  try {
    return new BOMInputStream(new FileInputStream(file),
      ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE);
  } catch (FileNotFoundException e) {
    throw new IllegalStateException("File not found: " + file.getAbsolutePath(), e);
  }
}

private CSVParser createCsvParser(String inputFileName, String delimiter) throws IOException {
    BufferedReader reader = new BufferedReader(new InputStreamReader(new BOMInputStream(new FileInputStream(inputFileName)), Charsets.UTF_8));
    CSVFormat format = CSVFormat.newFormat(delimiter.charAt(0))
            .withSkipHeaderRecord()
            .withIgnoreEmptyLines()
            .withAllowMissingColumnNames()
            .withQuote('"')
            .withHeader();
    return new CSVParser(reader, format);
}

public SubtitleFile(String fileName, String fileContents, String stylesStr) throws IOException, FatalParsingException {
  LOGGER.debug("Parsing subtitle file {}", fileName);

  TimedTextFileFormat timedTextFormat;
  switch (FilenameUtils.getExtension(fileName)) {
    case "ass":
      timedTextFormat = new FormatASS();
      break;
    case "srt":
      timedTextFormat = new FormatSRT();
      break;
    default:
      LOGGER.error("invalid subtitle file extension file: {}", fileName);
      throw new UnexpectedError();
  }

  // Convert String to InputStream to match subtitleFile API
  byte[] byteData = fileContents.getBytes("UTF-8");
  // Must use BOMInputStream otherwise files with BOM will broke :(((
  // => http://stackoverflow.com/questions/4897876/reading-utf-8-bom-marker
  try (BOMInputStream inputStream = new BOMInputStream(new ByteArrayInputStream(byteData))) {
    timedText = timedTextFormat.parseFile(fileName, inputStream, StandardCharsets.UTF_8);
  }

  if (timedText.warnings.length() > "List of non fatal errors produced during parsing:\n\n".length()) {
    LOGGER.warn("There was some warnings during parsing. See logs.");
    LOGGER.debug("Got warnings: {}", "\n" + timedText.warnings);
  }

  styles = parseStyles(stylesStr);
  timedText.styling = styles;
  timedText.description = JIJIMAKU_SIGNATURE;
  annotationCaptions = new TreeMap<>();

  // Initialization: add jijimaku mark and set style to Default
  addJijimakuMark();
  timedText.captions.values().stream().forEach(c -> c.style = styles.get("Default"));

  captionIter = timedText.captions.entrySet().iterator();
}

/**
 * Writes the content into a file using same format as source file
 *
 * @param content content to be written
 * @param path path to the file
 * @param sourceFileMatch
 * @throws CommandException
 */
public void writeFileContent(String content, Path path, FileMatch sourceFileMatch) throws CommandException {
    try {
        File outputFile = path.toFile();
        BOMInputStream inputStream = new BOMInputStream(FileUtils.openInputStream(sourceFileMatch.getPath().toFile()), false, boms);
        if (inputStream.hasBOM()) {
            FileUtils.writeByteArrayToFile(outputFile, inputStream.getBOM().getBytes());
            FileUtils.writeByteArrayToFile(outputFile, content.getBytes(inputStream.getBOMCharsetName()), true);
        } else {
            FileUtils.writeStringToFile(outputFile, content, StandardCharsets.UTF_8);
        }
    } catch (IOException e) {
        throw new CommandException("Cannot write file content in path: " + path.toString(), e);
    }
}

/**
 * Read the first bytes of the given file and try to determine the file
 * format. Read up to 100 KB before giving up.
 * @param f the file to read
 * @return the file format (or <code>null</code> if the format
 * could not be determined)
 * @throws IOException if the input stream could not be read
 */
public static String detect(File f) throws IOException {
  if (!f.exists()) {
    return null;
  }
  try (BufferedInputStream bis = new BufferedInputStream(new BOMInputStream(
      new FileInputStream(f)))) {
    return determineFileFormat(bis);
  }
}

private Path saveFile(MultipartFile file) throws IOException {
    // Get the filename and build the local file path
    String filename = file.getOriginalFilename();
    String directory = System.getProperty("java.io.tmpdir");
    String filepath = Paths.get(directory, filename).toString();

    // Save the file locally
    try (BufferedOutputStream stream =
                 new BufferedOutputStream(new FileOutputStream(new File(filepath)));
         BOMInputStream bis = new BOMInputStream(file.getInputStream(), false)) {
        IOUtils.copy(bis, stream);
    }
    return Paths.get(filepath);
}

public FileDownloaderWithBOM(WebzInputStreamDownloader downloader, String defaultEncoding) throws IOException, WebzException {

        this.bomIn = (BOMInputStream) new BOMInputStream(downloader.getInputStream(), false, ALL_BOMS);
        this.downloader = new FileDownloader(downloader.getFileSpecific(), bomIn);
        ByteOrderMark bom = bomIn.getBOM();

        if (bom == null) {
            actualEncoding = defaultEncoding;
            actualNumberOfBytes = downloader.getFileSpecific().getNumberOfBytes();
        } else {
            actualEncoding = bom.getCharsetName();
            actualNumberOfBytes = downloader.getFileSpecific().getNumberOfBytes() - bom.length();
        }
        reader = new InputStreamReader(bomIn, actualEncoding);
    }

public static InputStreamReader getInputStreamReader(File file, String encoding) throws IOException {

    FileInputStream fis = new FileInputStream(file);
    logger.debug("Reading file: " + file + " using encoding: " + encoding);
    BOMInputStream bis = new BOMInputStream(fis); //So that we can remove the BOM
    return new InputStreamReader(bis, encoding);
}

/**
 * Read the provided meta descriptor (e.g. meta.xml) and return a {@link Archive}.
 * @param metaDescriptor
 * @throws SAXException
 * @throws IOException
 * @throws UnsupportedArchiveException
 * @return a new {@link Archive}, never null
 */
public static Archive fromMetaDescriptor(InputStream metaDescriptor) throws SAXException, IOException, UnsupportedArchiveException {
  Archive archive = new Archive();
  try (BOMInputStream bomInputStream = new BOMInputStream(metaDescriptor)) {
    SAXParser p = SAX_FACTORY.newSAXParser();
    MetaXMLSaxHandler mh = new MetaXMLSaxHandler(archive);
    p.parse(bomInputStream, mh);
  } catch (ParserConfigurationException e) {
    throw new SAXException(e);
  }
  return archive;
}

private void extractMimeType(ZipEntry entry) {
  try {
    InputStream zipFileInputStream = getZipEntryInputStream(entry);
    BOMInputStream bomInputStream = new BOMInputStream(zipFileInputStream);
    DSSDocument document = new InMemoryDocument(bomInputStream);
    mimeType = StringUtils.trim(IOUtils.toString(getDocumentBytes(document), "UTF-8"));
    extractAsicEntry(entry, document);
  } catch (IOException e) {
    logger.error("Error parsing container mime type: " + e.getMessage());
    throw new TechnicalException("Error parsing container mime type: " + e.getMessage(), e);
  }
}

/**
 * Tries to fetch POM model from maven central for a given dependency
 * @param dependency dependency to fetch model to
 * @return POM model if found and valid
 * @throws IOException
 * @throws XmlPullParserException
 */
private static Model fetchModel(RawDependency dependency)
        throws IOException, XmlPullParserException {

    // Get the url to the POM file for this artifact
    String url = "http://central.maven.org/maven2/"
            + dependency.groupID.replace('.', '/') + '/' + dependency.artifactID + '/'
            + dependency.version + '/' + dependency.artifactID + '-' + dependency.version + ".pom";
    InputStream input = new BOMInputStream(new URL(url).openStream());

    MavenXpp3Reader xpp3Reader = new MavenXpp3Reader();
    Model model = xpp3Reader.read(input);
    input.close();
    return model;
}

@Test
public void testBOMInputStream_ParserWithInputStream() throws IOException {
    try (final BOMInputStream inputStream = createBOMInputStream("CSVFileParser/bom.csv");
            final CSVParser parser = CSVParser.parse(inputStream, UTF_8, CSVFormat.EXCEL.withHeader())) {
        for (final CSVRecord record : parser) {
            final String string = record.get("Date");
            Assert.assertNotNull(string);
            // System.out.println("date: " + record.get("Date"));
        }
    }
}

private static Charset charsetFor(BOMInputStream bis) throws IOException {
    ByteOrderMark bom = bis.getBOM();
    if (ByteOrderMark.UTF_8.equals(bom)) {
        return StandardCharsets.UTF_8;
    }
    if (ByteOrderMark.UTF_16LE.equals(bom)) {
        return StandardCharsets.UTF_16LE;
    }
    if (ByteOrderMark.UTF_16BE.equals(bom)) {
        return StandardCharsets.UTF_16BE;
    }
    return StandardCharsets.UTF_8;
}

public void run(File file, Seq seq, IDatabase database) throws Exception {
    this.seq = seq;
    this.database = database;
    CsvPreference pref = new CsvPreference.Builder('"', ';', "\n").build();
    try (FileInputStream fis = new FileInputStream(file);
            // exclude the byte order mark, if there is any
            BOMInputStream bom = new BOMInputStream(fis, false,
                    ByteOrderMark.UTF_8);
            InputStreamReader reader = new InputStreamReader(bom, "utf-8");
            BufferedReader buffer = new BufferedReader(reader);
            CsvListReader csvReader = new CsvListReader(buffer, pref)) {
        importFile(csvReader, database);
    }
}

private static CsvListReader createReader(InputStream stream)
        throws Exception {
    CsvPreference pref = new CsvPreference.Builder('"', ';', "\n").build();
    // exclude the byte order mark, if there is any
    BOMInputStream bom = new BOMInputStream(stream, false,
            ByteOrderMark.UTF_8);
    InputStreamReader reader = new InputStreamReader(bom, "utf-8");
    BufferedReader buffer = new BufferedReader(reader);
    CsvListReader csvReader = new CsvListReader(buffer, pref);
    return csvReader;
}

/**
 * Detects any BOMs and returns the corresponding charset
 */
private static String getCharsetFromBOM(final byte[] byteData) {
    BOMInputStream bomIn = new BOMInputStream(new ByteArrayInputStream(
            byteData));
    try {
        ByteOrderMark bom = bomIn.getBOM();
        if (bom != null) {
            return bom.getCharsetName();
        }
    } catch (IOException e) {
        return null;
    }
    return null;
}

String[] readFieldNamesFromFile( String fileName, CsvInputMeta csvInputMeta ) throws KettleException {
  String delimiter = environmentSubstitute( csvInputMeta.getDelimiter() );
  String enclosure = environmentSubstitute( csvInputMeta.getEnclosure() );
  String realEncoding = environmentSubstitute( csvInputMeta.getEncoding() );

  try ( FileObject fileObject = KettleVFS.getFileObject( fileName, getTransMeta() );
      BOMInputStream inputStream =
          new BOMInputStream( KettleVFS.getInputStream( fileObject ), ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE,
              ByteOrderMark.UTF_16BE ) ) {
    InputStreamReader reader = null;
    if ( Utils.isEmpty( realEncoding ) ) {
      reader = new InputStreamReader( inputStream );
    } else {
      reader = new InputStreamReader( inputStream, realEncoding );
    }
    EncodingType encodingType = EncodingType.guessEncodingType( reader.getEncoding() );
    String line =
        TextFileInput.getLine( log, reader, encodingType, TextFileInputMeta.FILE_FORMAT_UNIX, new StringBuilder(
            1000 ) );
    String[] fieldNames =
        CsvInput.guessStringsFromLine( log, line, delimiter, enclosure, csvInputMeta.getEscapeCharacter() );
    if ( !Utils.isEmpty( csvInputMeta.getEnclosure() ) ) {
      removeEnclosure( fieldNames, csvInputMeta.getEnclosure() );
    }
    trimFieldNames( fieldNames );
    return fieldNames;
  } catch ( IOException e ) {
    throw new KettleFileException( BaseMessages.getString( PKG, "CsvInput.Exception.CreateFieldMappingError" ), e );
  }
}

/**
 * Process a text-based Sitemap. Text sitemaps only list URLs but no priorities, last mods, etc.
 * 
 * @param content
 * @throws IOException
 */
private SiteMap processText(byte[] content, String sitemapUrl) throws IOException {

    logger.debug("Processing textual Sitemap");

    SiteMap textSiteMap = new SiteMap(sitemapUrl);
    textSiteMap.setType(SitemapType.TEXT);

    BOMInputStream bomIs = new BOMInputStream(new ByteArrayInputStream(content));
    @SuppressWarnings("resource")
    BufferedReader reader = new BufferedReader(new InputStreamReader(bomIs));

    String line;

    int i = 1;
    while ((line = reader.readLine()) != null) {
        if (line.length() > 0 && i <= MAX_URLS) {
            try {
                URL url = new URL(line);
                boolean valid = urlIsLegal(textSiteMap.getBaseUrl(), url.toString());

                if (valid || !strict) {
                    if (logger.isDebugEnabled()) {
                        StringBuffer sb = new StringBuffer("  ");
                        sb.append(i).append(". ").append(url);
                        logger.debug(sb.toString());
                    }
                    i++;
                    SiteMapURL surl = new SiteMapURL(url, valid);
                    textSiteMap.addSiteMapUrl(surl);
                }
            } catch (MalformedURLException e) {
                logger.debug("Bad URL [" + line + "].");
            }
        }
    }
    textSiteMap.setProcessed(true);
    return textSiteMap;
}

private void loadFile(File file) {
  try (FileInputStream inputStream = new FileInputStream(file)) {
    BOMInputStream bomIn = new BOMInputStream(inputStream, false,
            ByteOrderMark.UTF_8, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE,
            ByteOrderMark.UTF_32BE, ByteOrderMark.UTF_32LE);
    String charsetName;
    if (bomIn.hasBOM()) {
      bom = bomIn.getBOM();
      charsetName = bom.getCharsetName();
    } else {
      // No BOM found
      bom = null;
      charsetName = null;
    }
    String fileContents = StringTools.readStream(bomIn, charsetName);
    textArea.setText(fileContents);
    currentFile = file;
    updateTitle();
    if(recentFiles.contains(file.getAbsolutePath())) {
      recentFiles.remove(file.getAbsolutePath());
    }
    recentFiles.add(file.getAbsolutePath());
    localStorage.saveProperty("recentFiles", recentFiles);
    updateRecentFilesMenu();
  } catch (IOException e) {
    Tools.showError(e);
  }
}

private InputStreamReader getInputStreamReader(String filename, String encoding) throws IOException {
  String charsetName = encoding != null ? encoding : Charset.defaultCharset().name();
  InputStream is = System.in;
  if (!isStdIn(filename)) {
    is = new FileInputStream(new File(filename));
    BOMInputStream bomIn = new BOMInputStream(is, true, ByteOrderMark.UTF_8,
      ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE,
      ByteOrderMark.UTF_32BE,ByteOrderMark.UTF_32LE);
    if (bomIn.hasBOM() && encoding == null) {
      charsetName = bomIn.getBOMCharsetName();
    }
    is = bomIn;
  }
  return new InputStreamReader(new BufferedInputStream(is), charsetName);
}

private TypedInputStream openConnectionCheckRedirects(URLConnection c) throws IOException {
    boolean redir;
    int redirects = 0;
    InputStream in = null;
    String contentType = null;
    String contentEncoding = null;
    do {
        if (c instanceof HttpURLConnection) {
            ((HttpURLConnection) c).setInstanceFollowRedirects(false);
        }
        // We want to open the input stream before getting headers
        // because getHeaderField() et al swallow IOExceptions.
        in = new BufferedInputStream(new BOMInputStream(c.getInputStream()));
        contentType = c.getContentType();
        contentEncoding = c.getContentEncoding();
        redir = false;
        if (c instanceof HttpURLConnection) {
            HttpURLConnection http = (HttpURLConnection) c;
            int stat = http.getResponseCode();
            if (stat >= 300 && stat <= 307 && stat != 306
                    && stat != HttpURLConnection.HTTP_NOT_MODIFIED) {
                URL base = http.getURL();
                String loc = http.getHeaderField("Location");
                URL target = null;
                if (loc != null) {
                    target = new URL(base, loc);
                }
                http.disconnect();
                // Redirection should be allowed only for HTTP and HTTPS
                // and should be limited to 5 redirections at most.
                if (target == null
                        || !(target.getProtocol().equals("http") || target.getProtocol().equals("https"))
                        || c.getURL().getProtocol().equals("https") && target.getProtocol().equals("http")
                        || redirects >= 5) {
                    throw new SecurityException("illegal URL redirect");
                }
                redir = true;
                c = target.openConnection();
                redirects++;
            }
        }
    } while (redir);
    if(contentType==null) {
        contentType = "text/plain";
    }
    return new TypedInputStream(in, contentType, contentEncoding);
}

public static DataTable loadCSV(String fileName, String formatType, VariableType[] colTypesOverride, String[] colNamesOverride, boolean hasHeaderRow) {
    try {
        // use apache commons io + csv to load but convert to list of String[]
        // byte-order markers are handled if present at start of file.
        FileInputStream fis = new FileInputStream(fileName);
        final Reader reader = new InputStreamReader(new BOMInputStream(fis), "UTF-8");
        CSVFormat format;
        if ( formatType==null ) {
            format = hasHeaderRow ? CSVFormat.RFC4180.withHeader() : CSVFormat.RFC4180;
        }
        else {
            switch ( formatType.toLowerCase() ) {
                case "tsv":
                    format = hasHeaderRow ? CSVFormat.TDF.withHeader() : CSVFormat.TDF;
                    break;
                case "mysql":
                    format = hasHeaderRow ? CSVFormat.MYSQL.withHeader() : CSVFormat.MYSQL;
                    break;
                case "excel":
                    format = hasHeaderRow ? CSVFormat.EXCEL.withHeader() : CSVFormat.EXCEL;
                    break;
                case "rfc4180":
                default:
                    format = hasHeaderRow ? CSVFormat.RFC4180.withHeader() : CSVFormat.RFC4180;
                    break;
            }
        }
        final CSVParser parser = new CSVParser(reader, format);
        List<String[]> rows = new ArrayList<>();
        int numHeaderNames = parser.getHeaderMap().size();
        try {
            for (final CSVRecord record : parser) {
                String[] row = new String[record.size()];
                for (int j = 0; j<record.size(); j++) {
                    row[j] = record.get(j);
                }
                rows.add(row);
            }
        }
        finally {
            parser.close();
            reader.close();
        }

        VariableType[] actualTypes = computeColTypes(rows, numHeaderNames);

        Set<String> colNameSet = parser.getHeaderMap().keySet();
        String[] colNames = colNameSet.toArray(new String[colNameSet.size()]);
        if ( colNamesOverride!=null ) {
            colNames = colNamesOverride;
        }
        if ( colTypesOverride!=null ) {
            actualTypes = colTypesOverride;
        }
        return fromStrings(rows, actualTypes, colNames, false);
    }
    catch (Exception e) {
        throw new IllegalArgumentException("Can't open and/or read "+fileName, e);
    }
}

public static DataTable loadCSV(String fileName, VariableType[] colTypes, boolean hasHeaderRow) {
        int numCols = colTypes.length;
        try {
            final FileInputStream fis = new FileInputStream(fileName);
            final Reader r = new InputStreamReader(new BOMInputStream(fis), "UTF-8");
            final BufferedReader bf = new BufferedReader(r);
            List<int[]> rows = new ArrayList<>();
            String line;
            String[] colNames = null;
            if ( hasHeaderRow ) {
                line=bf.readLine();
                if ( line!=null ) {
                    line = line.trim();
                    if ( line.length()>0 ) {
                        colNames = line.split(",");
                        for (int i = 0; i<colNames.length; i++) {
                            colNames[i] = colNames[i].trim();
                        }
                    }
                }
            }
            int n = 0;
            while ( (line=bf.readLine())!=null ) {
                if ( n>0 && n % 10000 == 0 ) System.out.println(n);
                line = line.trim();
                if ( line.length()==0 ) continue;
                int[] row = new int[numCols];
                int comma = line.indexOf(',', 0);
                int prev = 0;
                int col = 0;
                while ( comma>=0 ) {
                    String v = line.substring(prev, comma);
                    row[col] = getValue(colTypes[col], v);

                    prev = comma+1;
                    comma = line.indexOf(',', comma+1);
                    col++;
                }
                // grab last element after last comma
                String lastv = line.substring(prev, line.length());
                row[col] = getValue(colTypes[col], lastv);

//              System.out.println();
                rows.add(row);
                n++;
            }

            DataTable data = new DataTable(rows, colTypes, colNames, null);
            return data;
        }
        catch (IOException ioe) {
            throw new IllegalArgumentException("Can't open and/or read "+fileName, ioe);
        }
    }

private List<RefCCDAValidationResult> runValidators(String validationObjective, String referenceFileName,
                                                    MultipartFile ccdaFile) throws SAXException, Exception {
    List<RefCCDAValidationResult> validatorResults = new ArrayList<>();
    InputStream ccdaFileInputStream = null;
    try {
        ccdaFileInputStream = ccdaFile.getInputStream();
        String ccdaFileContents = IOUtils.toString(new BOMInputStream(ccdaFileInputStream));

        List<RefCCDAValidationResult> mdhtResults = doMDHTValidation(validationObjective, referenceFileName, ccdaFileContents);
        if(mdhtResults != null && !mdhtResults.isEmpty()) {
            logger.info("Adding MDHT results");
            validatorResults.addAll(mdhtResults);
        }

        boolean isSchemaErrorInMdhtResults = mdhtResultsHaveSchemaError(mdhtResults);
        boolean isObjectiveAllowingVocabularyValidation = objectiveAllowsVocabularyValidation(validationObjective);
        if (!isSchemaErrorInMdhtResults && isObjectiveAllowingVocabularyValidation) {
            List<RefCCDAValidationResult> vocabResults = doVocabularyValidation(validationObjective, referenceFileName, ccdaFileContents);
            if(vocabResults != null && !vocabResults.isEmpty()) {
                logger.info("Adding Vocabulary results");
                validatorResults.addAll(vocabResults);
            }

            if(objectiveAllowsContentValidation(validationObjective)) {
             List<RefCCDAValidationResult> contentResults = doContentValidation(validationObjective, referenceFileName, ccdaFileContents);
            if(contentResults != null && !contentResults.isEmpty()) {
                logger.info("Adding Content results");
                validatorResults.addAll(contentResults);
            }
            } else {
                logger.info("Skipping Content validation due to: "
                        + "validationObjective (" + (validationObjective != null ? validationObjective : "null objective") 
                        + ") is not relevant or valid for Content validation");                 
            }
        } else {
            String separator = !isObjectiveAllowingVocabularyValidation && isSchemaErrorInMdhtResults ? " and " : "";
            logger.info("Skipping Vocabulary (and thus Content) validation due to: " 
                    + (isObjectiveAllowingVocabularyValidation ? "" : "validationObjective POSTed: " 
                    + (validationObjective != null ? validationObjective : "null objective") + separator) 
                    + (isSchemaErrorInMdhtResults ? "C-CDA Schema error(s) found" : ""));
        }
    } catch (IOException e) {
        throw new RuntimeException("Error getting CCDA contents from provided file", e);
    }finally {
        closeFileInputStream(ccdaFileInputStream);
    }
    return validatorResults;
}

/**
 * The main entry point into an Entity.Loader. Interprets each row of a CSV file within a zip file as a sinle
 * GTFS entity, and loads them into a table.
 *
 * @param zip the zip file from which to read a table
 */
public void loadTable(ZipFile zip) throws IOException {
    ZipEntry entry = zip.getEntry(tableName + ".txt");
    if (entry == null) {
        Enumeration<? extends ZipEntry> entries = zip.entries();
        // check if table is contained within sub-directory
        while (entries.hasMoreElements()) {
            ZipEntry e = entries.nextElement();
            if (e.getName().endsWith(tableName + ".txt")) {
                entry = e;
                feed.errors.add(new TableInSubdirectoryError(tableName, entry.getName().replace(tableName + ".txt", "")));
            }
        }
        /* This GTFS table did not exist in the zip. */
        if (this.isRequired()) {
            feed.errors.add(new MissingTableError(tableName));
        } else {
            LOG.info("Table {} was missing but it is not required.", tableName);
        }

        if (entry == null) return;
    }
    LOG.info("Loading GTFS table {} from {}", tableName, entry);
    InputStream zis = zip.getInputStream(entry);
    // skip any byte order mark that may be present. Files must be UTF-8,
    // but the GTFS spec says that "files that include the UTF byte order mark are acceptable"
    InputStream bis = new BOMInputStream(zis);
    CsvReader reader = new CsvReader(bis, ',', Charset.forName("UTF8"));
    this.reader = reader;
    boolean hasHeaders = reader.readHeaders();
    if (!hasHeaders) {
        feed.errors.add(new EmptyTableError(tableName));
    }
    while (reader.readRecord()) {
        // reader.getCurrentRecord() is zero-based and does not include the header line, keep our own row count
        if (++row % 500000 == 0) {
            LOG.info("Record number {}", human(row));
        }
        loadOneRow(); // Call subclass method to produce an entity from the current row.
    }
    if (row == 0) {
        feed.errors.add(new EmptyTableError(tableName));
    }
}

@Override
public BufferedReader create(Resource resource, String encoding)
        throws UnsupportedEncodingException, IOException {
    BOMInputStream bomInputStream = new BOMInputStream(resource.getInputStream());
    return new BufferedReader(new InputStreamReader(bomInputStream, encoding));
}

@Override
public BufferedReader create(Resource resource, String encoding)
        throws UnsupportedEncodingException, IOException {
    BOMInputStream bomInputStream = new BOMInputStream(resource.getInputStream());
    return new BufferedReader(new InputStreamReader(bomInputStream, encoding));
}

public static InputStreamReader getInputStreamReader(InputStream is, String encoding) throws IOException {

    logger.debug("Reading stream: using encoding: " + encoding);
    BOMInputStream bis = new BOMInputStream(is); //So that we can remove the BOM
    return new InputStreamReader(bis, encoding);
}