static private TableD readEmbeddedTableD(String location) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(location, "r")) { MessageScanner scan = new MessageScanner(raf); TableLookup lookup = scan.getTableLookup(); if (lookup != null) { return lookup.getLocalTableD(); } return null; } }
static private TableB readEmbeddedTableB(String location) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(location, "r")) { MessageScanner scan = new MessageScanner(raf); TableLookup lookup = scan.getTableLookup(); if (lookup != null) { return lookup.getLocalTableB(); } return null; } }
public int scanBufrFile(String filename, Counter total) throws Exception { int count = 0; try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; try { if (showMess) out.format("%sMessage %d header=%s%n", indent, count, m.getHeader()); count++; Counter counter = new Counter(); processBufrMessageAsDataset(scan, m, counter); if (showMess) out.format("%scount=%d miss=%d%n", indent, counter.nvals, counter.nmiss); total.add(counter); } catch (Exception e) { System.out.printf(" BARF:%s on %s%n", e.getMessage(), m.getHeader()); indent.setIndentLevel(0); } } } return count; }
public void scanBufrFile(String filename) throws Exception { int count = 0; RandomAccessFile raf = null; try { raf = new RandomAccessFile(filename, "r"); MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; out.format("%sMessage %d header=%s%n", indent, count++, m.getHeader()); processBufrMessageAsDataset(scan, m); } } finally { if (raf != null) raf.close(); } }
@Override public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask) throws IOException { super.open(raf, ncfile, cancelTask); scanner = new MessageScanner(raf); protoMessage = scanner.getFirstDataMessage(); if (protoMessage == null) throw new IOException("No data messages in the file= "+ncfile.getLocation()); // DataDescriptor dds = protoMessage.getRootDataDescriptor(); // construct the data descriptors, check for complete tables if (!protoMessage.isTablesComplete()) throw new IllegalStateException("BUFR file has incomplete tables"); // just get the fields config = BufrConfig.openFromMessage(raf, protoMessage, iospParam); // this fills the netcdf object Construct2 construct = new Construct2(protoMessage, config, ncfile); obsStructure = construct.getObsStructure(); ncfile.finish(); isSingle = false; }
public static void main(String arg[]) throws Exception { //String filename = "C:/temp/cache/uniqueMessages.bufr"; String filename = "C:/data/formats/bufr/uniqueExamples.bufr"; int size = 0; int count = 0; try (RandomAccessFile raf = new RandomAccessFile(filename, "r"); OutputStream out = new FileOutputStream("C:/data/formats/bufr/uniqueE/" + count + ".xml")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message message = scan.next(); if (message == null || !message.isTablesComplete() || !message.isBitCountOk()) continue; byte[] mbytes = scan.getMessageBytesFromLast(message); NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp"); NetcdfDataset ncd = new NetcdfDataset(ncfile); new Bufr2Xml(message, ncd, out, true); out.close(); count++; size += message.getMessageSize(); } } catch (Throwable e) { e.printStackTrace(); } System.out.printf("total size= %f Kb %n", .001 * size); } }
public int scanBufrFile(String filename, Counter total) throws Exception { int count = 0; RandomAccessFile raf = null; try { raf = new RandomAccessFile(filename, "r"); MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; try { if (showMess) out.format("%sMessage %d header=%s%n", indent, count, m.getHeader()); count++; Counter counter = new Counter(); processBufrMessageAsDataset(scan, m, counter); if (showMess) out.format("%scount=%d miss=%d%n", indent, counter.nvals, counter.nmiss); total.add(counter); } catch (Exception e) { System.out.printf(" BARF:%s on %s%n", e.getMessage(), m.getHeader()); indent.setIndentLevel(0); } } } finally { if (raf != null) raf.close(); } return count; }
static void writeUniqueDDS(String filename, WritableByteChannel wbc) throws IOException { System.out.printf("open %s ",filename); try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (m == null) { bad_msgs++; System.out.printf("Bad Message%n"); continue; } if (!messSet.contains(m)) { scan.writeCurrentMessage(wbc); messSet.add(m); } count++; } System.out.printf(" read = %d%n ", count); } }
static void extractNMessages(String filein, int n, String fileout) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(filein, "r"); FileOutputStream fos = new FileOutputStream(fileout); WritableByteChannel wbc = fos.getChannel()) { MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext() && (count < n)) { Message m = scan.next(); scan.writeCurrentMessage(wbc); count++; } } }
static void extractAllWithHeader(String filein, Pattern p, WritableByteChannel wbc) throws IOException { System.out.println("extract "+filein); try (RandomAccessFile raf = new RandomAccessFile(filein, "r")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); Matcher matcher = p.matcher(m.getHeader()); if (matcher.matches()) { scan.writeCurrentMessage(wbc); System.out.println(" found match " + m.getHeader()); } } } }
static void extractMessageByListhash(String filein, int want, String fileout) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(filein, "r"); FileOutputStream fos = new FileOutputStream(fileout); WritableByteChannel wbc = fos.getChannel()) { int count = 0; MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); int listHash = m.dds.getDataDescriptors().hashCode(); if (listHash == want) { scan.writeCurrentMessage(wbc); wbc.close(); raf.close(); System.out.printf("output %d from %s %n", count, filein); return; } count++; } } }
static void extractNthMessage(String filein, int msgno, String fileout) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(filein, "r"); FileOutputStream fos = new FileOutputStream(fileout); WritableByteChannel wbc = fos.getChannel()) { MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (msgno == count) { scan.writeCurrentMessage(wbc); wbc.close(); raf.close(); return; } count++; } } }
static void extractFirstMessageWithHeader(String filein, String header, String fileout) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(filein, "r"); FileOutputStream fos = new FileOutputStream(fileout); WritableByteChannel wbc = fos.getChannel()) { MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (m.getHeader().contains(header)) { scan.writeCurrentMessage(wbc); wbc.close(); raf.close(); return; } count++; } } }
static void scanMixedMessageTypes(String filename) throws IOException { //RandomAccessFile raf = new RandomAccessFile("C:\\data\\bufr\\edition3\\idd\\radiosonde\\SoundingVerticalRadiosonde4.bufr", "r"); try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { out.format("%n-----%nOpen %s size = %d Kb %n", raf.getLocation(), raf.length() / 1000); MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; List<String> desc = m.dds.getDescriptors(); String key = mixedSet.get(desc); if (null == key) { out.format(" new Message Type msg=%d <%s> ndesc=%d hashCode=%d%n", count, m.getHeader(), desc.size(), desc.hashCode()); m.getRootDataDescriptor(); m.dump(out); mixedSet.put(desc, filename); } else if (!key.equals(filename)) { out.format(" msg type from file= %s, hashcode=%d%n", key, desc.hashCode()); } count++; } out.format("nmsgs= %d nobs = %d%n", count, scan.getTotalObs()); } }
static void scanTimes(String filename) throws IOException { long start = System.nanoTime(); try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { out.format("%nOpen %s size = %d Kb %n", raf.getLocation(), raf.length() / 1000); MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; //if (count == 0) new BufrDump2().dump(out, m); out.format(" %s time=%s%n", m.getHeader(), m.getReferenceTime()); count++; } } }
public void execute(String filename) throws IOException { try (RandomAccessFile mraf = new RandomAccessFile(filename, "r")) { MessageScanner scanner = new MessageScanner(mraf); while (scanner.hasNext()) { Message m = scanner.next(); if (m == null) continue; total_msgs++; if (m.getNumberDatasets() == 0) continue; // LOOK check on tables complete etc ?? m.setRawBytes(scanner.getMessageBytes(m)); // decide what to do with the message dispatcher.dispatch(m); } dispatcher.resetBufrTableMessages(); } }
private void scanBufrFile(RandomAccessFile raf) throws Exception { NetcdfFile ncd = null; countObs = 0; try { MessageScanner scanner = new MessageScanner(raf); Message protoMessage = scanner.getFirstDataMessage(); if (protoMessage == null) throw new IOException("No message found!"); messHash = protoMessage.hashCode(); standardFields = StandardFields.extract(protoMessage); rootConverter = new FieldConverter(protoMessage.ids.getCenterId(), protoMessage.getRootDataDescriptor()); if (standardFields.hasStation()) { hasStations = true; map = new HashMap<>(1000); } featureType = guessFeatureType(standardFields); hasDate = standardFields.hasTime(); //ncd = NetcdfDataset.openDataset(raf.getLocation(), BufrIosp2.enhance, -1, null, null); // LOOK opening another raf ncd = NetcdfFile.open(raf.getLocation()); // LOOK opening another raf Attribute centerAtt = ncd.findGlobalAttribute(BufrIosp2.centerId); int center = (centerAtt == null) ? 0 : centerAtt.getNumericValue().intValue(); Sequence seq = (Sequence) ncd.findVariable(null, BufrIosp2.obsRecord); extract = new StandardFields.StandardFieldsFromStructure(center, seq); StructureDataIterator iter = seq.getStructureIterator(); processSeq( iter, rootConverter, true); setStandardActions(rootConverter); } finally { if (ncd != null) ncd.close(); } System.out.printf("nobs = %d%n", countObs); }
private boolean scanBufrFile(String filename) throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException { int count = 0; try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; byte[] mbytes = scan.getMessageBytes(m); try( NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp2")) { NetcdfDataset ncd = new NetcdfDataset(ncfile); } } } return true; }
static void scanReader(String filein) throws IOException { Formatter f = new Formatter(System.out); try (RandomAccessFile raf = new RandomAccessFile(filein, "r")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); m.dumpHeader(out); if (!m.dds.isCompressed()) { MessageUncompressedDataReader reader = new MessageUncompressedDataReader(); reader.readData(null, m, raf, null, false, null); } else { MessageCompressedDataReader reader = new MessageCompressedDataReader(); reader.readData(null, m, raf, null, f); } int nbitsGiven = 8 * (m.dataSection.getDataLength() - 4); System.out.printf("nbits counted = %d expected=%d %n", m.msg_nbits, nbitsGiven); System.out.printf("nbytes counted = %d expected=%d %n", m.getCountedDataBytes(), m.dataSection.getDataLength()); if (m.isTablesComplete() && !m.isBitCountOk()) System.out.printf("BAD BIT COUNT %n%n"); } } }
static void scanDDS(String filename) throws IOException { long start = System.nanoTime(); try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { out.format("\nOpen %s size = %d Kb \n", raf.getLocation(), raf.length() / 1000); MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; m.dump(out); if (!m.isTablesComplete()) { out.format("**INCOMPLETE "); } long startPos = m.is.getStartPos(); out.format(" msg= %d time=%s starts=%d len=%d end=%d dataEnd=%d\n", count, m.getReferenceTime(), startPos, m.is.getBufrLength(), (startPos + m.is.getBufrLength()), (m.dataSection.getDataPos() + m.dataSection.getDataLength())); out.format(" ndatasets=%d isCompressed=%s datatype=0x%x header=%s\n", m.getNumberDatasets(), m.dds.isCompressed(), m.dds.getDataType(), m.getHeader()); count++; break; } } }