@Override public int compareTo(DocumentURI o) { return uri.compareTo(o.getUri()); }
@Override public boolean equals(Object uri) { if (uri instanceof DocumentURI) { return this.uri.equals(((DocumentURI)uri).getUri()); } return false; }
protected static String getPathFromURI(DocumentURI uri) { String uriStr = uri.getUri(); try { URI child = new URI(uriStr); String childPath; if (child.isOpaque()) { childPath = child.getSchemeSpecificPart(); } else { childPath = child.getPath(); } return childPath; } catch (Exception ex) { LOG.error("Error parsing URI " + uriStr + "."); return uriStr; } } }
public static String getPathFromURI(DocumentURI uri) { String uriStr = uri.getUri(); try { URI child = new URI(uriStr); String childPath; if (child.isOpaque()) { childPath = child.getSchemeSpecificPart(); } else { childPath = child.getPath(); } return childPath; } catch (Exception ex) { LOG.warn("Error parsing URI " + uriStr + "."); return uriStr; } } }
/** * If outputDir is available and valid, modify DocumentURI, and return uri * in string * * @param key * @param outputDir * @return URI */ public static String getUriWithOutputDir(DocumentURI key, String outputDir){ String uri = key.getUri(); if (outputDir != null && !outputDir.isEmpty()) { uri = outputDir.endsWith("/") || uri.startsWith("/") ? outputDir + uri : outputDir + '/' + uri; key.setUri(uri); key.validate(); } return uri; }
public void map(DocumentURI uri, VALUE fileContent, Context context) throws IOException, InterruptedException { synchronized(readCount) { readCount.increment(1); } if (uri == null) { return; } else if (uri.getUri().isEmpty()) { LOG.warn("Skipped record: " + uri); return; } synchronized(attemptedCount) { attemptedCount.increment(1); } context.write(uri, fileContent); }
public static int getPlacementId(DocumentURI uri, int size) { switch (size) { case 0: throw new IllegalArgumentException("getPlacementId(size = 0)"); case 1: return 0; default: String nk = normalize(uri.getUri()); BigInteger uriKey = getUriKey(nk); long u = uriKey.longValue(); for (int i = 8; i <= 56; i += 8) { u += rotl(uriKey, i); } long v = (0xffff + size) / size; return (int) ((u & 0xffff) / v); } }
private int getBucketPlacementId(DocumentURI uri, int[][] buckets, int numBuckets, int numForests, int uForests) { String nk = LegacyAssignmentPolicy.normalize(uri.getUri()); BigInteger uriKey=LegacyAssignmentPolicy.getUriKey(nk); long u = uriKey.longValue(); for (int i = 14; i <= 56; i += 14) { u ^= uriKey.shiftRight(i).longValue(); } int bucket = (int) (u&0x3fff); int fIdx = buckets[numForests - 1][bucket]; boolean allUpdatble = numForests == uForests; if (!allUpdatble) { int[] partv = new int[uForests]; int j = 0; for (int i = 0; i < numForests; i++) { if (isUpdatable(i)) { partv[j++] = i; } } fIdx = partv[LegacyAssignmentPolicy.getPlacementId(uri, uForests)]; } return fIdx; }
public static void main(String args[]) throws IOException { System.out.println("Sequence File Creator"); String uri = args[0]; // output sequence file name Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(uri), conf); Path path = new Path(uri); SequenceFile.Writer writer = null; SimpleSequenceFileKey key = new SimpleSequenceFileKey(); SimpleSequenceFileValue<BytesWritable> value = new SimpleSequenceFileValue<BytesWritable>(); try { BytesWritable bw = new BytesWritable(); byte byteArray[] = {2,3,4}; bw.set(byteArray, 0, byteArray.length); writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass()); key.setDocumentURI(new DocumentURI("ABC")); value.setValue(bw); writer.append(key, value); System.err.println(key.getDocumentURI().getUri() + value); } finally { IOUtils.closeStream(writer); } } }
value.setValue(bw); writer.append(key, value); System.err.println(key.getDocumentURI().getUri() + " loaded."); } finally { IOUtils.closeStream(writer);
setKey(((SequenceFileKey) seqKey).getDocumentURI().getUri(), 0, 0, false); if (value instanceof Text) {
String syncSeen = reader.syncSeen() ? "*" : ""; System.out.printf("[%s%s]\t%s\t%s\n", position, syncSeen, ((SimpleSequenceFileKey) key).getDocumentURI().getUri(), ((SimpleSequenceFileValue) value).getValue()); position = reader.getPosition();
@Override public void write(DocumentURI uri, MarkLogicNode record) throws IOException, InterruptedException { // initialize recordString String recordString = record == null ? "()" : record.toString(); // execute query Session session = getSession(); try { AdhocQuery request = session.newAdhocQuery(query); request.setNewStringVariable(DOCURI_VARIABLE_NAME, InternalUtilities.unparse(uri.getUri())); request.setNewVariable(NODE_VARIABLE_NAME, ValueType.ELEMENT, recordString); session.submitRequest(request); commitIfNecessary(); } catch (RequestException e) { LOG.error(e); LOG.error(query); throw new IOException(e); } }
protected Content createContent(DocumentURI key, VALUEOUT value) throws IOException { String uri = key.getUri(); Content content = null; if (value instanceof Text) {
uri.getUri(); if (zipEntryName == null) { if (isExportDoc) {
return 0; default: String nk = normalize(uri.getUri()); BigInteger uriKey = getUriKey(nk); long u = uriKey.longValue();