/** * Filter a snapshot, provided via the InputStream, to each of the OutputStreams. * * The number of provided OutputStreams should be equal to the number of provided HollowFilterConfigs at instantiation. * * @param in the input stream to the snapshot * @param out the output streams to write the filtered snapshot * @throws IOException if the snapshot cannot be filtered */ public void filterSnapshot(InputStream in, OutputStream... out) throws IOException { filter(false, in, out); }
List<HollowSchema> filteredSchemaList = getFilteredSchemaList(unfilteredSchemaList, streamAndFilter.getConfig()); header.setSchemas(filteredSchemaList); headerWriter.writeHeader(header, streamAndFilter.getStream()); int numShards = readNumShards(in); HollowObjectTypeReadState.discardType(dis, (HollowObjectSchema)schema, numShards, delta); else copyFilteredObjectState(delta, dis, streamsWithType, (HollowObjectSchema)schema, numShards); } else { for(int j=0;j<streamsWithType.length;j++) { HollowListTypeReadState.discardType(dis, numShards, delta); else copyListState(delta, dis, streamsOnly(streamsWithType), numShards); } else if(schema instanceof HollowSetSchema) { if(streamsWithType.length == 0) HollowSetTypeReadState.discardType(dis, numShards, delta); else copySetState(delta, dis, streamsOnly(streamsWithType), numShards); } else if(schema instanceof HollowMapSchema) { if(streamsWithType.length == 0) HollowMapTypeReadState.discardType(dis, numShards, delta); else copyMapState(delta, dis, streamsOnly(streamsWithType), numShards);
HollowObjectSchema filteredObjectSchema = getFilteredObjectSchema(schema, streamAndFilters[i].getConfig()); filteredObjectSchemas[i] = filteredObjectSchema; filteredObjectSchema.writeTo(streamAndFilters[i].getStream()); long bitsPerRecord = writeBitsPerField(schema, bitsPerField, filteredObjectSchemas[i], streamAndFilters[i].getStream()); copySnapshotPopulatedOrdinals(is, os);
filterConfig.addType("NonexistentType"); FilteredHollowBlobWriter blobWriter = new FilteredHollowBlobWriter(filterConfig); ByteArrayOutputStream filteredBlobStream = new ByteArrayOutputStream(); blobWriter.filterSnapshot(new ByteArrayInputStream(snapshotData), filteredBlobStream); blobWriter.filterDelta(new ByteArrayInputStream(deltaData), filteredBlobStream); blobWriter.filterDelta(new ByteArrayInputStream(removeOnlyDeltaData), filteredBlobStream);
private void copyListState(boolean delta, DataInputStream is, DataOutputStream[] os, int numShards) throws IOException { if(numShards > 1) copyVInt(is, os); for(int shard=0;shard<numShards;shard++) { copyVInt(is, os); /// maxOrdinal if(delta) { GapEncodedVariableLengthIntegerReader.copyEncodedDeltaOrdinals(is, os); GapEncodedVariableLengthIntegerReader.copyEncodedDeltaOrdinals(is, os); } copyVInt(is, os); /// bitsPerListPointer copyVInt(is, os); /// bitsPerElement copyVLong(is, os); /// totalNumberOfElements copySegmentedLongArray(is, os); copySegmentedLongArray(is, os); } if(!delta) copySnapshotPopulatedOrdinals(is, os); }
private HollowSchema getFilteredSchema(HollowSchema schema, HollowFilterConfig filterConfig) { if(filterConfig.doesIncludeType(schema.getName())) { if(schema.getSchemaType() == SchemaType.OBJECT) return getFilteredObjectSchema((HollowObjectSchema) schema, filterConfig); return schema; } return null; }
private void copySetState(boolean delta, DataInputStream is, DataOutputStream[] os, int numShards) throws IOException { if(numShards > 1) copyVInt(is, os); for(int shard=0;shard<numShards;shard++) { copyVInt(is, os); /// max ordinal if(delta) { GapEncodedVariableLengthIntegerReader.copyEncodedDeltaOrdinals(is, os); GapEncodedVariableLengthIntegerReader.copyEncodedDeltaOrdinals(is, os); } copyVInt(is, os); /// bitsPerSetPointer copyVInt(is, os); /// bitsPerSetSizeValue copyVInt(is, os); /// bitsPerElement copyVLong(is, os); /// totalNumberOfBuckets copySegmentedLongArray(is, os); copySegmentedLongArray(is, os); } if(!delta) copySnapshotPopulatedOrdinals(is, os); }
private void copyMapState(boolean delta, DataInputStream is, DataOutputStream[] os, int numShards) throws IOException { if(numShards > 1) copyVInt(is, os); for(int shard=0;shard<numShards;shard++) { copyVInt(is, os); /// max ordinal if(delta) { GapEncodedVariableLengthIntegerReader.copyEncodedDeltaOrdinals(is, os); GapEncodedVariableLengthIntegerReader.copyEncodedDeltaOrdinals(is, os); } copyVInt(is, os); /// bitsPerMapPointer copyVInt(is, os); /// bitsPerMapSizeValue copyVInt(is, os); /// bitsPerKeyElement copyVInt(is, os); /// bitsPerValueElement copyVLong(is, os); /// totalNumberOfBuckets copySegmentedLongArray(is, os); copySegmentedLongArray(is, os); } if(!delta) copySnapshotPopulatedOrdinals(is, os); }
/** * Filter a delta (or reversedelta), provided via the InputStream, to each of the OutputStreams. * * The number of provided OutputStreams should be equal to the number of provided HollowFilterConfigs at instantiation. * * @param in the input stream to the delta * @param out the output streams to write the filtered delta * @throws IOException if the delta cannot be filtered */ public void filterDelta(InputStream in, OutputStream... out) throws IOException { filter(true, in, out); }