@Override public Stream<String> list() throws IOException { // This prevents write/delete/merge/purge operations to occur ONLY for the time needed to // retrieve (non-merged/non-deleted) small files and retrieve an iterator over Lucene // index. This DOES NOT PROTECT the iteration over the Lucene index, thus changes to the // file store during the iteration may be reflected in the iteration results this.lock.readLock().lock(); try { // Check active flag Preconditions.checkState(this.active.get()); // Retrieve small files final List<String> smallNames = new ArrayList<>(); for (final FileStatus fs : this.fileSystem.listStatus(this.smallFilesPath)) { final String smallName = fs.getPath().getName(); if (indexGet(smallName) != null) { smallNames.add(smallName); } } // Retrieve an iterator over zipped files final Iterator<String> zippedNames = indexList(false); // Return the concatenation of the two return Stream.concat(Stream.create(smallNames), Stream.create(zippedNames)); } finally { // Always release the lock this.lock.readLock().unlock(); } }
@Override public Stream<Record> lookup(final URI type, final Set<? extends URI> ids, final Set<? extends URI> properties) throws IOException, IllegalArgumentException, IllegalStateException { return Stream.concat(Stream.create(ids).chunk(64) .transform(new Function<List<? extends URI>, Stream<Record>>() { @Override public Stream<Record> apply(final List<? extends URI> input) { final StringBuilder builder = new StringBuilder(); builder.append(" VALUES ?s {"); for (final URI id : input) { builder.append(" <").append(id.toString()).append(">"); } builder.append(" }"); try { return query(builder.toString(), type, properties, null); } catch (final IOException ex) { throw Throwables.propagate(ex); } } }, 1)); }
private static Stream<Record> download(final Session session, final boolean dumpResources, final boolean dumpMentions, @Nullable final String id) throws Throwable { final List<URI> types = Lists.newArrayList(); if (dumpResources) { types.add(KS.RESOURCE); } if (dumpMentions) { types.add(KS.MENTION); } return Stream.concat(Stream.create(types) .transform( (final URI type) -> { LOGGER.info("Downloading {} data", type.getLocalName().toLowerCase()); try { final Retrieve retrieve = session.retrieve(type) .limit((long) Integer.MAX_VALUE) .timeout(7 * 24 * 60 * 60 * 1000L); // 1 week if (id != null) { retrieve.ids(new URIImpl(id)); } return retrieve.exec(); // return session.retrieve(type).limit((long) Integer.MAX_VALUE) // .timeout(24 * 60 * 60 * 1000L).exec(); } catch (final Throwable ex) { throw Throwables.propagate(ex); } }, 1)); }