public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { ByteArrayOutputStream os = new ByteArrayOutputStream(); IOUtils.copy(stream, os); String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY); if (name == null) { name = ""; } try { NetcdfFile ncFile = NetcdfFile.openInMemory(name, os.toByteArray()); unravelStringMet(ncFile, null, metadata); } catch (IOException e) { throw new TikaException("HDF parse error", e); } XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); xhtml.endDocument(); }
/** * Read a remote CDM file into memory. All reads are then done from memory. * * @param uri location of CDM file, must be accessible through url.toURL().openStream(). * @return a NetcdfFile, which is completely in memory * @throws IOException if error reading file */ public static NetcdfFile openInMemory(URI uri) throws IOException { URL url = uri.toURL(); byte[] contents = IO.readContentsToByteArray(url.openStream()); return openInMemory(uri.toString(), contents); }
/** * Read a remote CDM file into memory. All reads are then done from memory. * * @param uri location of CDM file, must be accessible through url.toURL().openStream(). * @return a NetcdfFile, which is completely in memory * @throws IOException if error reading file */ public static NetcdfFile openInMemory(URI uri) throws IOException { URL url = uri.toURL(); byte[] contents = IO.readContentsToByteArray(url.openStream()); return openInMemory(uri.toString(), contents); }
/** * Read a remote CDM file into memory. All reads are then done from memory. * * @param uri location of CDM file, must be accessible through url.toURL().openStream(). * @return a NetcdfFile, which is completely in memory * @throws IOException if error reading file */ public static NetcdfFile openInMemory(URI uri) throws IOException { URL url = uri.toURL(); byte[] contents = IO.readContentsToByteArray(url.openStream()); return openInMemory(uri.toString(), contents); }
/** * Read a local CDM file into memory. All reads are then done from memory. * * @param filename location of CDM file, must be a local file. * @return a NetcdfFile, which is completely in memory * @throws IOException if error reading file */ public static NetcdfFile openInMemory(String filename) throws IOException { File file = new File(filename); ByteArrayOutputStream bos = new ByteArrayOutputStream((int) file.length()); InputStream in = new BufferedInputStream(new FileInputStream(filename)); IO.copy(in, bos); return openInMemory(filename, bos.toByteArray()); }
/** * Read a local CDM file into memory. All reads are then done from memory. * * @param filename location of CDM file, must be a local file. * @return a NetcdfFile, which is completely in memory * @throws IOException if error reading file */ public static NetcdfFile openInMemory(String filename) throws IOException { File file = new File(filename); ByteArrayOutputStream bos = new ByteArrayOutputStream((int) file.length()); try (InputStream in = new BufferedInputStream(new FileInputStream(filename))) { IO.copy(in, bos); } return openInMemory(filename, bos.toByteArray()); }
/** * Read a local CDM file into memory. All reads are then done from memory. * * @param filename location of CDM file, must be a local file. * @return a NetcdfFile, which is completely in memory * @throws IOException if error reading file */ public static NetcdfFile openInMemory(String filename) throws IOException { File file = new File(filename); ByteArrayOutputStream bos = new ByteArrayOutputStream((int) file.length()); try (InputStream in = new BufferedInputStream(new FileInputStream(filename))) { IO.copy(in, bos); } return openInMemory(filename, bos.toByteArray()); }
@Override public NetcdfFile getNetcdfFile() { try { netcdfFile = NetcdfFile.openInMemory(getClass().getResource(file).toURI()); return netcdfFile; } catch (Exception e) { throw new RuntimeException(e); } }
private NetcdfFile openInMemory( String filename) throws IOException { String pathname = TestDir.cdmLocalTestDataDir +filename; System.out.println("**** OpenInMemory "+pathname); byte[] ba = IO.readFileToByteArray( pathname); NetcdfFile ncfile = NetcdfFile.openInMemory("OpenInMemory", ba); System.out.println(ncfile); return ncfile; }
private void processBufrMessageAsDataset(MessageScanner scan, Message m, Counter counter) throws Exception { byte[] mbytes = scan.getMessageBytes(m); NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp"); Sequence obs = (Sequence) ncfile.findVariable(BufrIosp2.obsRecord); StructureDataIterator sdataIter = obs.getStructureIterator(-1); processSequence(obs, sdataIter, counter); }
private void processBufrMessageAsDataset(MessageScanner scan, Message m, Counter counter) throws Exception { byte[] mbytes = scan.getMessageBytes(m); NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp"); Sequence obs = (Sequence) ncfile.findVariable(BufrIosp.obsRecord); StructureDataIterator sdataIter = obs.getStructureIterator(-1); processSequence(obs, sdataIter, counter); }
@Test public void testExtractStartTime_NoTimeInfo() throws Exception { final URL resource = getClass().getResource("test_without_time_info.nc"); final NetcdfFile netcdfFile = NetcdfFile.openInMemory(resource.toURI()); ProductData.UTC startTime = BinnedProductReader.extractStartTime(netcdfFile); assertNull(startTime); }
@Test public void testExtractEndTime_NoTimeInfo() throws Exception { final URL resource = getClass().getResource("test_without_time_info.nc"); final NetcdfFile netcdfFile = NetcdfFile.openInMemory(resource.toURI()); ProductData.UTC endTime = BinnedProductReader.extractEndTime(netcdfFile); assertNull(endTime); }
@Test public void testExtractEndTime() throws Exception { final URL resource = getClass().getResource("test.nc"); final NetcdfFile netcdfFile = NetcdfFile.openInMemory(resource.toURI()); ProductData.UTC startTime = BinnedProductReader.extractEndTime(netcdfFile); assertEquals(ProductData.UTC.parse("20030101:0000", "yyyyMMdd:HHmm").getAsDate().getTime(), startTime.getAsDate().getTime()); }
@Test public void testExtractStartTime() throws Exception { final URL resource = getClass().getResource("test.nc"); final NetcdfFile netcdfFile = NetcdfFile.openInMemory(resource.toURI()); ProductData.UTC startTime = BinnedProductReader.extractStartTime(netcdfFile); assertEquals(ProductData.UTC.parse("20030101:0000", "yyyyMMdd:HHmm").getAsDate().getTime(), startTime.getAsDate().getTime()); }
private void processBufrMessageAsDataset(MessageScanner scan, Message m) throws Exception { byte[] mbytes = scan.getMessageBytes(m); NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp"); NetcdfDataset ncd = new NetcdfDataset(ncfile); SequenceDS obs = (SequenceDS) ncd.findVariable(BufrIosp.obsRecord); StructureDataIterator sdataIter = obs.getStructureIterator(-1); //writeSequence(obs, sdataIter); extractFirst(sdataIter, new Extract()); }
@Test public void checkGrid() throws Exception { String endpoint = TestOnLocalServer.withHttpPath("/ncss/grid/gribCollection/GFS_CONUS_80km/GFS_CONUS_80km_20120227_0000.grib1?var=Temperature_isobaric"); byte[] content = TestOnLocalServer.getContent(endpoint, 200, ContentType.netcdf); // Open the binary response in memory try (NetcdfFile nf = NetcdfFile.openInMemory("test_data.nc", content)) { GridDataset gdsDataset = new GridDataset(new NetcdfDataset(nf)); assertNotNull(gdsDataset.findGridByName("Temperature_isobaric")); logger.debug("{}", nf); } }
@Test public void checkFmrcBest() throws Exception { String endpoint = TestOnLocalServer.withHttpPath("/ncss/grid/testGFSfmrc/GFS_CONUS_80km_nc_best.ncd?var=Relative_humidity_height_above_ground,Temperature_height_above_ground"); byte[] content = TestOnLocalServer.getContent(endpoint, 200, ContentType.netcdf); // Open the binary response in memory try (NetcdfFile nf = NetcdfFile.openInMemory("test_data.nc", content)) { GridDataset gdsDataset = new GridDataset(new NetcdfDataset(nf)); assertNotNull(gdsDataset.findGridByName("Relative_humidity_height_above_ground")); logger.debug("{}", nf); } }
@Test public void testNcssFailure() throws Exception{ String filename = "scanCdmUnitTests/formats/netcdf4/COMPRESS_LEV2_20140201000000-GLOBCURRENT-L4-CURekm_15m-ERAWS_EEM-v02.0-fv01.0.nc"; String endpoint = TestOnLocalServer.withHttpPath("/ncss/grid/"+filename+ "?var=eastward_ekman_current_velocity&north=79.8750&west=-140&east=170&south=-79.8750&horizStride=1&"+ "time_start=2014-02-01T00%3A00%3A00Z&time_end=2014-02-01T00%3A00%3A00Z&timeStride=1&accept=netcdf4"); byte[] content = TestOnLocalServer.getContent(endpoint, 200, ContentType.netcdf); // Open the binary response in memory try (NetcdfFile nf = NetcdfFile.openInMemory("test_data.nc", content)) { GridDataset gdsDataset = new GridDataset(new NetcdfDataset(nf)); assertNotNull(gdsDataset.findGridByName("eastward_ekman_current_velocity")); logger.debug("{}", nf); } } }
private boolean scanBufrFile(String filename) throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException { int count = 0; try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; byte[] mbytes = scan.getMessageBytes(m); try( NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp2")) { NetcdfDataset ncd = new NetcdfDataset(ncfile); } } } return true; }