@Override public Stream<Record> lookup(final URI type, final Set<? extends URI> ids, final Set<? extends URI> properties) throws IOException, IllegalArgumentException, IllegalStateException { return Stream.concat(Stream.create(ids).chunk(64) .transform(new Function<List<? extends URI>, Stream<Record>>() { @Override public Stream<Record> apply(final List<? extends URI> input) { final StringBuilder builder = new StringBuilder(); builder.append(" VALUES ?s {"); for (final URI id : input) { builder.append(" <").append(id.toString()).append(">"); } builder.append(" }"); try { return query(builder.toString(), type, properties, null); } catch (final IOException ex) { throw Throwables.propagate(ex); } } }, 1)); }
@Override public long count(URI type, @Nullable XPath condition) throws IOException, IllegalArgumentException, IllegalStateException { optimize(type); if (condition == null) { return readers.get(type).numDocs(); } else { Stream<Record> stream = retrieve(type, condition, null); return stream.count(); } }
@Override public void store(final URI type, final Record record) throws IOException, IllegalStateException { // Delete existing data for the record URI delete(type, record.getID()); // Add statements final List<Statement> statements = Record.encode(Stream.create(record), ImmutableList.of(type)).toList(); this.transaction.add(statements); }
@Override public synchronized long count(final URI type, @Nullable final XPath condition) throws IOException, IllegalArgumentException, IllegalStateException { Preconditions.checkState(!this.ended); final Map<URI, Record> table = this.getTable(type); return this.filter(Stream.create(table.values()), condition).count(); }
@Override public void add(final Iterable<? extends Statement> statements) throws IOException, IllegalStateException { if (LOGGER.isDebugEnabled()) { final AtomicLong count = new AtomicLong(); final AtomicBoolean eof = new AtomicBoolean(); @SuppressWarnings("unchecked") Iterable<Statement> stmts = (Iterable<Statement>) statements; final Stream<Statement> stream = Stream.create(stmts).track(count, eof); final long ts = System.currentTimeMillis(); super.add(stream); LOGGER.debug("{} - {} statements added in {} ms{}", this, count, System.currentTimeMillis() - ts, eof.get() ? ", EOF" : ""); } else { super.add(statements); } }
private static Stream<Record> download(final Session session, final boolean dumpResources, final boolean dumpMentions, @Nullable final String id) throws Throwable { final List<URI> types = Lists.newArrayList(); if (dumpResources) { types.add(KS.RESOURCE); } if (dumpMentions) { types.add(KS.MENTION); } return Stream.concat(Stream.create(types) .transform( (final URI type) -> { LOGGER.info("Downloading {} data", type.getLocalName().toLowerCase()); try { final Retrieve retrieve = session.retrieve(type) .limit((long) Integer.MAX_VALUE) .timeout(7 * 24 * 60 * 60 * 1000L); // 1 week if (id != null) { retrieve.ids(new URIImpl(id)); } return retrieve.exec(); // return session.retrieve(type).limit((long) Integer.MAX_VALUE) // .timeout(24 * 60 * 60 * 1000L).exec(); } catch (final Throwable ex) { throw Throwables.propagate(ex); } }, 1)); }
return Stream.create();
final Stream<BindingSet> bindingStream = Stream.create(this.transaction.query( SelectQuery.from(query), null, null)); stmtStream = bindingStream.transform(null, new Function<Handler<Statement>, Handler<BindingSet>>() { stmtStream = bindingStream.transform(new Function<BindingSet, Statement>() { recordStream = recordStream.filter(condition.asPredicate(), 1); recordStream = recordStream.transform(new Function<Record, Record>() {
private Stream<Record> select(final Map<URI, Record> table, final Stream<? extends URI> stream) { return stream.transform(new Function<URI, Record>() { @Override public Record apply(final URI id) { return table.get(id); } }, 0); }
@Override public Stream<String> list() throws IOException { // This prevents write/delete/merge/purge operations to occur ONLY for the time needed to // retrieve (non-merged/non-deleted) small files and retrieve an iterator over Lucene // index. This DOES NOT PROTECT the iteration over the Lucene index, thus changes to the // file store during the iteration may be reflected in the iteration results this.lock.readLock().lock(); try { // Check active flag Preconditions.checkState(this.active.get()); // Retrieve small files final List<String> smallNames = new ArrayList<>(); for (final FileStatus fs : this.fileSystem.listStatus(this.smallFilesPath)) { final String smallName = fs.getPath().getName(); if (indexGet(smallName) != null) { smallNames.add(smallName); } } // Retrieve an iterator over zipped files final Iterator<String> zippedNames = indexList(false); // Return the concatenation of the two return Stream.concat(Stream.create(smallNames), Stream.create(zippedNames)); } finally { // Always release the lock this.lock.readLock().unlock(); } }
tx.lookup(KS.MENTION, (Set) restrictions.get(KS.HAS_MENTION), ImmutableSet.of(KS.MENTION_OF)) .transform(URI.class, true, KS.MENTION_OF).toCollection(ids); tx.lookup(KS.RESOURCE, (Set) restrictions.get(KS.MENTION_OF), ImmutableSet.of(KS.HAS_MENTION)) .transform(URI.class, true, KS.HAS_MENTION).toCollection(ids);
@Override public void delete(final URI type, final URI id) throws IOException, IllegalStateException { // Obtain the statements to delete throuh a lookup final List<Statement> statements = Record.encode( lookup(type, ImmutableSet.of(id), null), ImmutableList.of(type)).toList(); // Perform the deletion this.transaction.remove(statements); }
private static void writeBinary(final Stream<Record> records, final File file) throws IOException { final String base = file.getAbsolutePath(); final Dictionary<URI> dictionary = Dictionary.createLocalDictionary(URI.class, new File( base + ".dict")); final Serializer serializer = new Serializer(false, dictionary, Statements.VALUE_FACTORY); final CountingOutputStream stream = new CountingOutputStream(IO.write(base + ".gz")); try { records.toHandler(new Handler<Record>() { private int records = 0; @Override public void handle(final Record record) throws Throwable { if (record == null || this.records > 0 && this.records % 1000 == 0) { LOGGER.info("{} records, {} bytes processed ({} bytes/record)", this.records, stream.getCount(), stream.getCount() / this.records); } if (record != null) { serializer.toStream(stream, record); ++this.records; } } }); } finally { IO.closeQuietly(records); IO.closeQuietly(stream); } }
private Stream<Record> filter(final Stream<Record> stream, @Nullable final XPath xpath) { if (xpath == null) { return stream; } return stream.filter(xpath.asPredicate(), 0); }
@Override public void close() { if (!this.state.compareAndSet(INITIALIZED, CLOSED) && !this.state.compareAndSet(NEW, CLOSED)) { return; } List<Stream<String>> streamsToEnd; synchronized (this.pendingListStreams) { streamsToEnd = Lists.newArrayList(this.pendingListStreams); } try { for (final Stream<String> stream : streamsToEnd) { try { LOGGER.warn("Forcing closure of stream due to FileStore closure"); stream.close(); } catch (final Throwable ex) { LOGGER.error("Exception caught while closing stream: " + ex.getMessage(), ex); } } } finally { super.close(); } }
@Override public Stream<String> list() throws IOException { return Stream.create(new HadoopIterator()); }
private static Stream<Record> upload(final Session session, final Criteria criteria, final Stream<Record> axioms) { return axioms.transform(null, new Function<Handler<Record>, Handler<Record>>() { @Override public Handler<Record> apply(final Handler<Record> handler) { return new UploadHandler(session, criteria, handler); } }); }