return Stream.create();
@Override public Stream<String> list() throws IOException { return Stream.create(new HadoopIterator()); }
@Override protected void doDelete(@Nullable final Long timeout, final URI type, @Nullable final XPath condition, @Nullable final Set<URI> ids, final Handler<? super Outcome> handler) throws Throwable { modify(new RecordUpdater() { @Override public Record computeNewRecord(final URI id, @Nullable final Record oldRecord, @Nullable final Record suppliedRecord) throws Throwable { assert oldRecord != null; assert suppliedRecord == null; return null; } }, type, condition, ids == null ? null : Stream.create(ids), handler); }
return Stream.create();
@Override protected void doUpdate(@Nullable final Long timeout, final URI type, @Nullable final XPath condition, @Nullable final Set<URI> ids, @Nullable final Record record, @Nullable final Criteria criteria, final Handler<? super Outcome> handler) throws Throwable { modify(new RecordUpdater() { @Override public Record computeNewRecord(final URI id, @Nullable final Record oldRecord, @Nullable final Record suppliedRecord) throws Throwable { assert oldRecord != null; assert suppliedRecord == null; final Record newRecord = Record.create(oldRecord, true); criteria.merge(newRecord, record); return newRecord; } }, type, condition, ids == null ? null : Stream.create(ids), handler); }
@Override public synchronized long count(final URI type, @Nullable final XPath condition) throws IOException, IllegalArgumentException, IllegalStateException { Preconditions.checkState(!this.ended); final Map<URI, Record> table = this.getTable(type); return this.filter(Stream.create(table.values()), condition).count(); }
@Override public synchronized Stream<Record> retrieve(final URI type, @Nullable final XPath condition, @Nullable final Set<? extends URI> properties) throws IOException, IllegalArgumentException, IllegalStateException { Preconditions.checkState(!this.ended); final Map<URI, Record> table = this.getTable(type); return this.project(this.filter(Stream.create(table.values()), condition), properties); }
@Override public Stream<String> list() throws IOException { // This prevents write/delete/merge/purge operations to occur ONLY for the time needed to // retrieve (non-merged/non-deleted) small files and retrieve an iterator over Lucene // index. This DOES NOT PROTECT the iteration over the Lucene index, thus changes to the // file store during the iteration may be reflected in the iteration results this.lock.readLock().lock(); try { // Check active flag Preconditions.checkState(this.active.get()); // Retrieve small files final List<String> smallNames = new ArrayList<>(); for (final FileStatus fs : this.fileSystem.listStatus(this.smallFilesPath)) { final String smallName = fs.getPath().getName(); if (indexGet(smallName) != null) { smallNames.add(smallName); } } // Retrieve an iterator over zipped files final Iterator<String> zippedNames = indexList(false); // Return the concatenation of the two return Stream.concat(Stream.create(smallNames), Stream.create(zippedNames)); } finally { // Always release the lock this.lock.readLock().unlock(); } }
@Override public void add(final Iterable<? extends Statement> statements) throws IOException, IllegalStateException { if (LOGGER.isDebugEnabled()) { final AtomicLong count = new AtomicLong(); final AtomicBoolean eof = new AtomicBoolean(); @SuppressWarnings("unchecked") Iterable<Statement> stmts = (Iterable<Statement>) statements; final Stream<Statement> stream = Stream.create(stmts).track(count, eof); final long ts = System.currentTimeMillis(); super.add(stream); LOGGER.debug("{} - {} statements added in {} ms{}", this, count, System.currentTimeMillis() - ts, eof.get() ? ", EOF" : ""); } else { super.add(statements); } }
@Override public void remove(final Iterable<? extends Statement> statements) throws IOException, IllegalStateException { if (LOGGER.isDebugEnabled()) { final AtomicLong count = new AtomicLong(); final AtomicBoolean eof = new AtomicBoolean(); @SuppressWarnings("unchecked") Iterable<Statement> stmts = (Iterable<Statement>) statements; final Stream<Statement> stream = Stream.create(stmts).track(count, eof); final long ts = System.currentTimeMillis(); super.remove(stream); LOGGER.debug("{} - {} statements removed in {} ms{}", this, count, System.currentTimeMillis() - ts, eof.get() ? ", EOF" : ""); } else { super.remove(statements); } }
@Override public synchronized Stream<Record> lookup(final URI type, final Set<? extends URI> ids, @Nullable final Set<? extends URI> properties) throws IOException, IllegalArgumentException, IllegalStateException { Preconditions.checkState(!this.ended); final Map<URI, Record> table = this.getTable(type); return this.project(this.select(table, Stream.create(ids)), properties); }
@Override public Stream<Record> retrieve(URI type, @Nullable XPath condition, @Nullable Set<? extends URI> properties) throws IOException, IllegalArgumentException, IllegalStateException { String tableName = getTableName(type); List<Record> returns = new ArrayList<>(); logger.debug("Retrieving all lines"); String query = selectAllQuery.replace("$tableName", tableName); try { Statement statement = con.createStatement(); ResultSet resultSet = statement.executeQuery(query); while (resultSet.next()) { Record r = unserializeRecord(resultSet.getBytes("value")); if (condition != null && !condition.evalBoolean(r)) { continue; } if (properties != null) { r.retain(Iterables.toArray(properties, URI.class)); } returns.add(r); } } catch (SQLException e) { throw new IOException(e); } return Stream.create(returns); }
@Override public Stream<Record> lookup(URI type, Set<? extends URI> ids, @Nullable Set<? extends URI> properties) throws IOException, IllegalArgumentException, IllegalStateException { String tableName = getTableName(type); List<Record> returns = new ArrayList<>(); for (URI id : ids) { String uri; try { uri = id.toString(); } catch (NullPointerException e) { throw new IOException(e); } logger.debug(String.format("Selecting %s", uri)); String query = selectQuery.replace("$tableName", tableName); try { PreparedStatement stmt = con.prepareStatement(query); stmt.setString(1, uri); ResultSet set = stmt.executeQuery(); while (set.next()) { Record r = unserializeRecord(set.getBytes("value")); if (properties != null) { r.retain(Iterables.toArray(properties, URI.class)); } returns.add(r); } } catch (SQLException e) { throw new IOException(e); } } return Stream.create(returns); }
private synchronized void update(final Map<URI, Map<URI, Record>> tables, final int revision) throws IOException { if (this.revision != revision) { throw new IOException("Commit failed due to concurrent modifications " + this.revision + ", " + revision); } OutputStream stream = null; try { stream = Files.writeWithBackup(this.fileSystem, this.filePath); final List<Record> records = Lists.newArrayList(); for (final URI type : tables.keySet()) { records.addAll(tables.get(type).values()); } final RDFFormat format = RDFFormat.forFileName(this.filePath.getName()); RDFUtil.writeRDF(stream, format, Data.getNamespaceMap(), null, Record.encode(Stream.create(records), ImmutableSet.<URI>of())); ++this.revision; this.tables = tables; MemoryDataStore.LOGGER.info("MemoryDataStore updated, {} records persisted", records.size()); } catch (final Throwable ex) { MemoryDataStore.LOGGER.error("MemoryDataStore update failed", ex); } finally { Util.closeQuietly(stream); } }
@Override public Stream<Record> retrieve(URI type, @Nullable XPath condition, @Nullable Set<? extends URI> properties) throws IOException, IllegalArgumentException, IllegalStateException { optimize(type); List<Record> returns = new ArrayList<>(); for (int i = 0; i < readers.get(type).numDocs(); i++) { Document doc = readers.get(type).document(i); Record r = unserializeRecord(doc.getBinaryValue(VALUE_NAME), serializer); if (condition != null && !condition.evalBoolean(r)) { continue; } if (properties != null) { r.retain(Iterables.toArray(properties, URI.class)); } returns.add(r); } return Stream.create(returns); }
@Override public Stream<Record> lookup(URI type, Set<? extends URI> ids, @Nullable Set<? extends URI> properties) throws IOException, IllegalArgumentException, IllegalStateException { optimize(type); List<Record> returns = new ArrayList<>(); for (URI id : ids) { String uri; try { uri = id.toString(); } catch (NullPointerException e) { throw new IOException(e); } logger.debug("Selecting {}", uri); Term s = new Term(KEY_NAME, uri); TermDocs termDocs = readers.get(type).termDocs(s); if (termDocs.next()) { Document doc = readers.get(type).document(termDocs.doc()); Record r = unserializeRecord(doc.getBinaryValue(VALUE_NAME), serializer); if (properties != null && !properties.isEmpty()) { r.retain(Iterables.toArray(properties, URI.class)); } returns.add(r); } } return Stream.create(returns); }
@Override public void handle(final Record record) throws Throwable { if (record == null || this.records > 0 && this.records % 1000 == 0) { LOGGER.info(this.records + " records, " + this.triples + " triples processed"); } if (record != null) { final List<Statement> statements = Record.encode(Stream.create(record), ImmutableSet.of()).toList(); writer.handleComment(record.getID().toString()); for (final Statement statement : statements) { writer.handleStatement(statement); } ++this.records; this.triples += statements.size(); } }
@Override public void store(final URI type, final Record record) throws IOException, IllegalStateException { // Delete existing data for the record URI delete(type, record.getID()); // Add statements final List<Statement> statements = Record.encode(Stream.create(record), ImmutableList.of(type)).toList(); this.transaction.add(statements); }
@Override public Stream<Record> lookup(final URI type, final Set<? extends URI> ids, final Set<? extends URI> properties) throws IOException, IllegalArgumentException, IllegalStateException { return Stream.concat(Stream.create(ids).chunk(64) .transform(new Function<List<? extends URI>, Stream<Record>>() { @Override public Stream<Record> apply(final List<? extends URI> input) { final StringBuilder builder = new StringBuilder(); builder.append(" VALUES ?s {"); for (final URI id : input) { builder.append(" <").append(id.toString()).append(">"); } builder.append(" }"); try { return query(builder.toString(), type, properties, null); } catch (final IOException ex) { throw Throwables.propagate(ex); } } }, 1)); }
private static Stream<Record> download(final Session session, final boolean dumpResources, final boolean dumpMentions, @Nullable final String id) throws Throwable { final List<URI> types = Lists.newArrayList(); if (dumpResources) { types.add(KS.RESOURCE); } if (dumpMentions) { types.add(KS.MENTION); } return Stream.concat(Stream.create(types) .transform( (final URI type) -> { LOGGER.info("Downloading {} data", type.getLocalName().toLowerCase()); try { final Retrieve retrieve = session.retrieve(type) .limit((long) Integer.MAX_VALUE) .timeout(7 * 24 * 60 * 60 * 1000L); // 1 week if (id != null) { retrieve.ids(new URIImpl(id)); } return retrieve.exec(); // return session.retrieve(type).limit((long) Integer.MAX_VALUE) // .timeout(24 * 60 * 60 * 1000L).exec(); } catch (final Throwable ex) { throw Throwables.propagate(ex); } }, 1)); }