public TableLookup getTableLookup() throws IOException { while (hasNext()) { next(); } return (embedTable != null) ? embedTable.getTableLookup() : null; }
public Message getFirstDataMessage() throws IOException { while (hasNext()) { Message m = next(); if (m == null) continue; if (m.containsBufrTable()) continue; // not data if (m.getNumberDatasets() == 0) continue; // empty return m; } return null; }
public int scanBufrFile(String filename, Counter total) throws Exception { int count = 0; try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; try { if (showMess) out.format("%sMessage %d header=%s%n", indent, count, m.getHeader()); count++; Counter counter = new Counter(); processBufrMessageAsDataset(scan, m, counter); if (showMess) out.format("%scount=%d miss=%d%n", indent, counter.nvals, counter.nmiss); total.add(counter); } catch (Exception e) { System.out.printf(" BARF:%s on %s%n", e.getMessage(), m.getHeader()); indent.setIndentLevel(0); } } } return count; }
private StructureDataIterator readNextMessage() throws IOException { if (!scanner.hasNext()) return null; Message m = scanner.next(); if (m == null) { log.warn("BUFR scanner hasNext() true but next() null!"); return null; } if (m.containsBufrTable()) // data messages only return readNextMessage(); // mixed messages if (!protoMessage.equals(m)) { if (messHash == null) messHash = new HashSet<>(20); if (!messHash.contains(m.hashCode())) { log.warn("File " + raf.getLocation() + " has different BUFR message types hash=" + protoMessage.hashCode() + "; skipping"); messHash.add(m.hashCode()); } return readNextMessage(); } ArrayStructure as = readMessage(m); return as.getStructureDataIterator(); }
public void scanBufrFile(String filename) throws Exception { int count = 0; RandomAccessFile raf = null; try { raf = new RandomAccessFile(filename, "r"); MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; out.format("%sMessage %d header=%s%n", indent, count++, m.getHeader()); processBufrMessageAsDataset(scan, m); } } finally { if (raf != null) raf.close(); } }
public static void main(String arg[]) throws Exception { //String filename = "C:/temp/cache/uniqueMessages.bufr"; String filename = "C:/data/formats/bufr/uniqueExamples.bufr"; int size = 0; int count = 0; try (RandomAccessFile raf = new RandomAccessFile(filename, "r"); OutputStream out = new FileOutputStream("C:/data/formats/bufr/uniqueE/" + count + ".xml")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message message = scan.next(); if (message == null || !message.isTablesComplete() || !message.isBitCountOk()) continue; byte[] mbytes = scan.getMessageBytesFromLast(message); NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp"); NetcdfDataset ncd = new NetcdfDataset(ncfile); new Bufr2Xml(message, ncd, out, true); out.close(); count++; size += message.getMessageSize(); } } catch (Throwable e) { e.printStackTrace(); } System.out.printf("total size= %f Kb %n", .001 * size); } }
public int scanBufrFile(String filename, Counter total) throws Exception { int count = 0; RandomAccessFile raf = null; try { raf = new RandomAccessFile(filename, "r"); MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; try { if (showMess) out.format("%sMessage %d header=%s%n", indent, count, m.getHeader()); count++; Counter counter = new Counter(); processBufrMessageAsDataset(scan, m, counter); if (showMess) out.format("%scount=%d miss=%d%n", indent, counter.nvals, counter.nmiss); total.add(counter); } catch (Exception e) { System.out.printf(" BARF:%s on %s%n", e.getMessage(), m.getHeader()); indent.setIndentLevel(0); } } } finally { if (raf != null) raf.close(); } return count; }
static void writeUniqueDDS(String filename, WritableByteChannel wbc) throws IOException { System.out.printf("open %s ",filename); try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (m == null) { bad_msgs++; System.out.printf("Bad Message%n"); continue; } if (!messSet.contains(m)) { scan.writeCurrentMessage(wbc); messSet.add(m); } count++; } System.out.printf(" read = %d%n ", count); } }
static void extractNMessages(String filein, int n, String fileout) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(filein, "r"); FileOutputStream fos = new FileOutputStream(fileout); WritableByteChannel wbc = fos.getChannel()) { MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext() && (count < n)) { Message m = scan.next(); scan.writeCurrentMessage(wbc); count++; } } }
static void extractAllWithHeader(String filein, Pattern p, WritableByteChannel wbc) throws IOException { System.out.println("extract "+filein); try (RandomAccessFile raf = new RandomAccessFile(filein, "r")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); Matcher matcher = p.matcher(m.getHeader()); if (matcher.matches()) { scan.writeCurrentMessage(wbc); System.out.println(" found match " + m.getHeader()); } } } }
static void extractMessageByListhash(String filein, int want, String fileout) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(filein, "r"); FileOutputStream fos = new FileOutputStream(fileout); WritableByteChannel wbc = fos.getChannel()) { int count = 0; MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); int listHash = m.dds.getDataDescriptors().hashCode(); if (listHash == want) { scan.writeCurrentMessage(wbc); wbc.close(); raf.close(); System.out.printf("output %d from %s %n", count, filein); return; } count++; } } }
static void extractNthMessage(String filein, int msgno, String fileout) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(filein, "r"); FileOutputStream fos = new FileOutputStream(fileout); WritableByteChannel wbc = fos.getChannel()) { MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (msgno == count) { scan.writeCurrentMessage(wbc); wbc.close(); raf.close(); return; } count++; } } }
static void extractFirstMessageWithHeader(String filein, String header, String fileout) throws IOException { try (RandomAccessFile raf = new RandomAccessFile(filein, "r"); FileOutputStream fos = new FileOutputStream(fileout); WritableByteChannel wbc = fos.getChannel()) { MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (m.getHeader().contains(header)) { scan.writeCurrentMessage(wbc); wbc.close(); raf.close(); return; } count++; } } }
static void scanMixedMessageTypes(String filename) throws IOException { //RandomAccessFile raf = new RandomAccessFile("C:\\data\\bufr\\edition3\\idd\\radiosonde\\SoundingVerticalRadiosonde4.bufr", "r"); try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { out.format("%n-----%nOpen %s size = %d Kb %n", raf.getLocation(), raf.length() / 1000); MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; List<String> desc = m.dds.getDescriptors(); String key = mixedSet.get(desc); if (null == key) { out.format(" new Message Type msg=%d <%s> ndesc=%d hashCode=%d%n", count, m.getHeader(), desc.size(), desc.hashCode()); m.getRootDataDescriptor(); m.dump(out); mixedSet.put(desc, filename); } else if (!key.equals(filename)) { out.format(" msg type from file= %s, hashcode=%d%n", key, desc.hashCode()); } count++; } out.format("nmsgs= %d nobs = %d%n", count, scan.getTotalObs()); } }
static void scanTimes(String filename) throws IOException { long start = System.nanoTime(); try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { out.format("%nOpen %s size = %d Kb %n", raf.getLocation(), raf.length() / 1000); MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; //if (count == 0) new BufrDump2().dump(out, m); out.format(" %s time=%s%n", m.getHeader(), m.getReferenceTime()); count++; } } }
public void execute(String filename) throws IOException { try (RandomAccessFile mraf = new RandomAccessFile(filename, "r")) { MessageScanner scanner = new MessageScanner(mraf); while (scanner.hasNext()) { Message m = scanner.next(); if (m == null) continue; total_msgs++; if (m.getNumberDatasets() == 0) continue; // LOOK check on tables complete etc ?? m.setRawBytes(scanner.getMessageBytes(m)); // decide what to do with the message dispatcher.dispatch(m); } dispatcher.resetBufrTableMessages(); } }
private boolean scanBufrFile(String filename) throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException { int count = 0; try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; byte[] mbytes = scan.getMessageBytes(m); try( NetcdfFile ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp2")) { NetcdfDataset ncd = new NetcdfDataset(ncfile); } } } return true; }
static void scanReader(String filein) throws IOException { Formatter f = new Formatter(System.out); try (RandomAccessFile raf = new RandomAccessFile(filein, "r")) { MessageScanner scan = new MessageScanner(raf); while (scan.hasNext()) { Message m = scan.next(); m.dumpHeader(out); if (!m.dds.isCompressed()) { MessageUncompressedDataReader reader = new MessageUncompressedDataReader(); reader.readData(null, m, raf, null, false, null); } else { MessageCompressedDataReader reader = new MessageCompressedDataReader(); reader.readData(null, m, raf, null, f); } int nbitsGiven = 8 * (m.dataSection.getDataLength() - 4); System.out.printf("nbits counted = %d expected=%d %n", m.msg_nbits, nbitsGiven); System.out.printf("nbytes counted = %d expected=%d %n", m.getCountedDataBytes(), m.dataSection.getDataLength()); if (m.isTablesComplete() && !m.isBitCountOk()) System.out.printf("BAD BIT COUNT %n%n"); } } }
static void scanDDS(String filename) throws IOException { long start = System.nanoTime(); try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) { out.format("\nOpen %s size = %d Kb \n", raf.getLocation(), raf.length() / 1000); MessageScanner scan = new MessageScanner(raf); int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue; m.dump(out); if (!m.isTablesComplete()) { out.format("**INCOMPLETE "); } long startPos = m.is.getStartPos(); out.format(" msg= %d time=%s starts=%d len=%d end=%d dataEnd=%d\n", count, m.getReferenceTime(), startPos, m.is.getBufrLength(), (startPos + m.is.getBufrLength()), (m.dataSection.getDataPos() + m.dataSection.getDataLength())); out.format(" ndatasets=%d isCompressed=%s datatype=0x%x header=%s\n", m.getNumberDatasets(), m.dds.isCompressed(), m.dds.getDataType(), m.getHeader()); count++; break; } } }
int count = 0; while (scan.hasNext()) { Message m = scan.next(); if (m == null) continue;