private static float getOverallProgress( final GeoWaveRowRange range, final GeoWaveInputKey currentKey, final ProgressPerRange progress) { final float rangeProgress = getProgressForRange(range, currentKey); return progress.getOverallProgress(rangeProgress); }
@Override public int compare(final IndexRangeLocation o1, final IndexRangeLocation o2) { return (o1.rangeLocationPair.getCardinality() - o2.rangeLocationPair.getCardinality()) < 0 ? -1 : 1; } });
protected SplitsProvider createSplitsProvider() { return new SplitsProvider(); }
protected static double getRangeLength(final GeoWaveRowRange range) { if ((range == null) || (range.getStartSortKey() == null) || (range.getEndSortKey() == null)) { return 1; } final byte[] start = range.getStartSortKey(); final byte[] end = range.getEndSortKey(); final int maxDepth = Math.max(end.length, start.length); final BigInteger startBI = new BigInteger(extractBytes(start, maxDepth)); final BigInteger endBI = new BigInteger(extractBytes(end, maxDepth)); return endBI.subtract(startBI).doubleValue(); }
protected static GeoWaveRowRange rangeIntersection( final GeoWaveRowRange thisRange, final GeoWaveRowRange otherRange) { final ByteArrayRange thisByteArrayRange = new ByteArrayRange(thisRange.getStartSortKey(), thisRange.getEndSortKey()); final ByteArrayRange otherByteArrayRange = new ByteArrayRange(otherRange.getStartSortKey(), otherRange.getEndSortKey()); final ByteArrayRange overlappingRange = thisByteArrayRange.intersection(otherByteArrayRange); return new GeoWaveRowRange( null, overlappingRange.getStart(), overlappingRange.getEnd(), true, false); } }
protected static BigInteger getStart(final GeoWaveRowRange range, final int cardinality) { final byte[] start = range.getStartSortKey(); byte[] startBytes; if (!range.isInfiniteStartSortKey() && (start != null)) { startBytes = extractBytes(start, cardinality); } else { startBytes = extractBytes(new byte[] {}, cardinality); } return new BigInteger(startBytes); }
protected static BigInteger getEnd(final GeoWaveRowRange range, final int cardinality) { final byte[] end = range.getEndSortKey(); byte[] endBytes; if (!range.isInfiniteStopSortKey() && (end != null)) { endBytes = extractBytes(end, cardinality); } else { endBytes = extractBytes(new byte[] {}, cardinality, true); } return new BigInteger(endBytes); }
private static float getProgressForRange( final GeoWaveRowRange range, final GeoWaveInputKey currentKey) { if (currentKey == null) { return 0f; } if ((range != null) && (range.getStartSortKey() != null) && (range.getEndSortKey() != null) && (currentKey.getGeoWaveKey() != null)) { // TODO GEOWAVE-1018 this doesn't account for partition keys at all // just look at the row progress return getProgressForRange( range.getStartSortKey(), range.getEndSortKey(), GeoWaveKey.getCompositeId(currentKey.getGeoWaveKey())); } // if we can't figure it out, then claim no progress return 0f; }
protected double getCardinality( final RowRangeHistogramStatistics<?> rangeStats, final GeoWaveRowRange range) { if (range == null) { if (rangeStats != null) { return rangeStats.getTotalCount(); } else { // with an infinite range and no histogram we have no info to // base a cardinality on return 0; } } return rangeStats == null ? 0.0 : rangeStats.cardinality(range.getStartSortKey(), range.getEndSortKey()); }
public DynamoDBReader( final RecordReaderParams recordReaderParams, final DynamoDBOperations operations, final boolean visibilityEnabled) { readerParams = null; this.recordReaderParams = recordReaderParams; processAuthorizations( recordReaderParams.getAdditionalAuthorizations(), (RangeReaderParams<T>) recordReaderParams); this.operations = operations; this.rowTransformer = (GeoWaveRowIteratorTransformer<T>) GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER; this.visibilityEnabled = visibilityEnabled; initRecordScanner(); }
public void readFields(final DataInput in) throws IOException, InstantiationException, IllegalAccessException { final boolean nullRange = in.readBoolean(); if (nullRange) { range = null; } else { range = new GeoWaveRowRange(); range.readFields(in); } location = in.readUTF(); cardinality = in.readDouble(); }
@Override public float getProgress() throws IOException { if ((numKeysRead > 0) && (currentGeoWaveKey == null)) { return 1.0f; } if (currentGeoWaveRangeIndexPair == null) { return 0.0f; } final ProgressPerRange progress = progressPerRange.get(currentGeoWaveRangeIndexPair); if (progress == null) { return Math.min( 1, Math.max( 0, getProgressForRange(currentGeoWaveRangeIndexPair.getRange(), currentGeoWaveKey))); } return Math.min( 1, Math.max( 0, getOverallProgress( currentGeoWaveRangeIndexPair.getRange(), currentGeoWaveKey, progress))); }
private synchronized double getTotalCardinality() { double sum = 0.0; for (final SplitInfo si : splitInfo.values()) { for (final RangeLocationPair pair : si.getRangeLocationPairs()) { sum += pair.getCardinality(); } } return sum; } }
protected static byte[] extractBytes( final byte[] seq, final int numBytes, final boolean infiniteEndKey) { final byte[] bytes = new byte[numBytes + 2]; bytes[0] = 1; bytes[1] = 0; for (int i = 0; i < numBytes; i++) { if (i >= seq.length) { if (infiniteEndKey) { // -1 is 0xff bytes[i + 2] = -1; } else { bytes[i + 2] = 0; } } else { bytes[i + 2] = seq[i]; } } return bytes; }
@Override public List<InputSplit> getSplits( final CommonQueryOptions commonOptions, final DataTypeQueryOptions<?> typeOptions, final IndexQueryOptions indexOptions, final QueryConstraints constraints, final TransientAdapterStore adapterStore, final AdapterIndexMappingStore aimStore, final DataStatisticsStore statsStore, final InternalAdapterStore internalAdapterStore, final IndexStore indexStore, final JobContext context, final Integer minSplits, final Integer maxSplits) throws IOException, InterruptedException { return splitsProvider.getSplits( baseOperations, commonOptions, typeOptions, indexOptions, constraints, adapterStore, statsStore, internalAdapterStore, indexStore, indexMappingStore, context, minSplits, maxSplits); } }
public void write(final DataOutput out) throws IOException { out.writeBoolean(range == null); if (range != null) { range.write(out); } out.writeUTF(location); out.writeDouble(cardinality); } }
@Override public GeoWaveRow next() { if (!hasNext()) { throw new NoSuchElementException(); } removeFrom = currentIterator; return currentIterator.next(); }
/** * This implementation of length is only an estimate, it does not provide exact values. Do not * have your code rely on this return value. */ @Override public long getLength() throws IOException { long diff = 0; for (final Entry<String, SplitInfo> indexEntry : splitInfo.entrySet()) { for (final RangeLocationPair range : indexEntry.getValue().getRangeLocationPairs()) { diff += (long) range.getCardinality(); } } return diff; }
private static float getProgressForRange( final byte[] start, final byte[] end, final byte[] position) { final int maxDepth = Math.min(Math.max(end.length, start.length), position.length); final BigInteger startBI = new BigInteger(SplitsProvider.extractBytes(start, maxDepth)); final BigInteger endBI = new BigInteger(SplitsProvider.extractBytes(end, maxDepth)); final BigInteger positionBI = new BigInteger(SplitsProvider.extractBytes(position, maxDepth)); return (float) (positionBI.subtract(startBI).doubleValue() / endBI.subtract(startBI).doubleValue()); }