private Map<URI, Record> extractRelated(final Record record) throws Throwable { // TODO: this has to be done better using some Schema object final URI id = record.getID(); final URI type = record.getSystemType(); final Map<URI, Record> map = Maps.newHashMap(); if (type.equals(KS.RESOURCE)) { for (final URI mentionID : record.get(KS.HAS_MENTION, URI.class)) { map.put(mentionID, Record.create(mentionID, KS.MENTION).add(KS.MENTION_OF, id)); } } else if (type.equals(KS.MENTION)) { final URI resourceID = record.getUnique(KS.MENTION_OF, URI.class); if (resourceID != null) { map.put(resourceID, Record.create(resourceID, KS.RESOURCE).add(KS.HAS_MENTION, id)); } } else { // TODO: handle entities, axioms and contexts throw new Error("Unexpected type: " + type); } return map; }
private void preprocess(final Record record) throws Throwable { // Ignore ks:storedAs possibly supplied by clients, as it is computed with file upload if (KS.RESOURCE.equals(record.getSystemType())) { record.set(KS.STORED_AS, null); } // TODO: add here filtering logic to be applied to records coming from the client }
@Override public void store(final URI type, final Record record) throws IOException, IllegalStateException { Preconditions.checkState(!this.ended); Preconditions.checkState(!this.readOnly); Preconditions.checkArgument(record.getID() != null); final Map<URI, Record> table = this.getTable(type); table.put(record.getID(), Record.create(record, true)); }
recordsToStore.add(newRecord); status = Status.OK_CREATED; } else if (!oldRecord.hash().equals(newRecord.hash())) { recordsToStore.add(newRecord); status = Status.OK_MODIFIED; final URI type = MoreObjects.firstNonNull(oldRel, newRel).getSystemType(); if (oldRel != null && newRel != null) { for (final URI property : oldRel.getProperties()) { final List<URI> newValues = newRel.get(property, URI.class); if (!newValues.isEmpty()) { final List<URI> oldValues = oldRel.get(property, URI.class); oldRel.remove(property, newValues); newRel.remove(property, oldValues); final List<URI> oldProperties = oldRel == null ? nilList : oldRel.getProperties(); final List<URI> newProperties = newRel == null ? nilList : newRel.getProperties(); .getUnique(); if (related == null) { related = Record.create(id, type); assert oldRel != null; if (!property.equals(RDF.TYPE)) { related.remove(property, oldRel.get(property)); assert newRel != null; if (!property.equals(RDF.TYPE)) { related.add(property, newRel.get(property));
final Record metadata = resource.getUnique(KS.STORED_AS, Record.class); if (metadata == null) { return null; final String fileName = metadata.getUnique(NFO.FILE_NAME, String.class); check(fileName != null, null, resourceID, "No filename stored for resource (!)"); final String fileTypeString = metadata.getUnique(NIE.MIME_TYPE, String.class); if (mimeTypes != null) { check(fileTypeString != null, Status.ERROR_NOT_ACCEPTABLE, resourceID, Data.mimeTypeToExtensions(transformToType), "bin"); final String name = MoreObjects.firstNonNull( metadata.getUnique(NFO.FILE_NAME, String.class, null), "download") + "." + ext; stream = transform(fileTypeString, transformToType, stream); final Representation representation = Representation.create(stream); final Record meta = representation.getMetadata(); meta.setID(metadata.getID()); meta.set(NIE.MIME_TYPE, transformToType); meta.set(NFO.FILE_NAME, name); meta.set(NFO.FILE_LAST_MODIFIED, metadata.getUnique(NFO.FILE_LAST_MODIFIED)); return representation; representation.getMetadata().setID(metadata.getID()); for (final URI property : metadata.getProperties()) { representation.getMetadata().set(property, metadata.get(property));
@Override public void store(final URI type, final Record record) throws IOException, IllegalStateException { // Delete existing data for the record URI delete(type, record.getID()); // Add statements final List<Statement> statements = Record.encode(Stream.create(record), ImmutableList.of(type)).toList(); this.transaction.add(statements); }
@Override public void delete(final URI type, final URI id) throws IOException, IllegalStateException { // Obtain the statements to delete throuh a lookup final List<Statement> statements = Record.encode( lookup(type, ImmutableSet.of(id), null), ImmutableList.of(type)).toList(); // Perform the deletion this.transaction.remove(statements); }
@Override public URI apply(final Record record) { return record.getID(); }
@Override public synchronized void init() throws IOException, IllegalStateException { Preconditions.checkState(!this.initialized && !this.closed); this.initialized = true; InputStream stream = null; try { if (this.fileSystem.exists(this.filePath)) { stream = Files.readWithBackup(this.fileSystem, this.filePath); final RDFFormat format = RDFFormat.forFileName(this.filePath.getName()); final List<Record> records = Record.decode( RDFUtil.readRDF(stream, format, null, null, false), ImmutableSet.of(KS.RESOURCE, KS.MENTION, KS.ENTITY, KS.CONTEXT), false) .toList(); for (final Record record : records) { final URI id = Preconditions.checkNotNull(record.getID()); final URI type = Preconditions.checkNotNull(record.getSystemType()); MemoryDataStore.this.tables.get(type).put(id, record); } MemoryDataStore.LOGGER.info("{} initialized, {} records loaded", this.getClass() .getSimpleName(), records.size()); } else { MemoryDataStore.LOGGER.info("{} initialized, no record loaded", this.getClass() .getSimpleName()); } } finally { Util.closeQuietly(stream); } }
} else if (record != CachingDataStore.NULL) { CachingDataStore.this.localHitCount.incrementAndGet(); result.add(Record.create(record, true)); // clone to preserve cached one CachingDataStore.this.globalHitCount.incrementAndGet(); result.add(Record.create(record, true)); // clone record result.add(Record.create(record, true)); localCache.put(record.getID(), record); missingIDs.remove(record.getID()); if (this.localRevision == CachingDataStore.this.globalRevision) { for (final Record record : fetched) { globalCache.put(record.getID(), record); for (final Record record : result) { final URI[] projection = properties.toArray(new URI[properties.size()]); record.retain(projection);
private Object encodeRecord(final Record record, @Nullable final Set<URI> propertiesToEncode) { final URI id = record.getID(); final Object encodedID = id == null ? null : encodeIdentifier(id); final List<Object> props = Lists.newArrayList(); for (final URI property : record.getProperties()) { if (propertiesToEncode == null || propertiesToEncode.contains(property)) { ensureInDictionary(property); final List<? extends Object> nodes = record.get(property); if (property.equals(RDF.TYPE)) { for (final Object value : nodes) { if (value instanceof URI) { ensureInDictionary((URI) value); } } } final GenericData.Record prop = new GenericData.Record(Schemas.PROPERTY); prop.put("propertyURI", encodeIdentifier(property)); prop.put("propertyValue", encodeNodes(nodes)); props.add(prop); } } return SerializerAvro.newGenericRecord(Schemas.RECORD, encodedID, props); }
final Record oldMetadata = resource.getUnique(KS.STORED_AS, Record.class); resource.set(KS.STORED_AS, null); metadata.setID(Data.getValueFactory().createURI(resourceID + "_file")); fileName = metadata.getUnique(NFO.FILE_NAME, String.class); String fileType = metadata.getUnique(NIE.MIME_TYPE, String.class); if (fileType != null) { try { } catch (final IllegalArgumentException ex) { metadata.set(NIE.MIME_TYPE, null); metadata.set(NFO.FILE_NAME, fileName); metadata.set(NIE.MIME_TYPE, fileType); final Record hash = Record.create(); hash.set(NFO.HASH_ALGORITHM, "MD5"); hash.set(NFO.HASH_VALUE, hos.hash().toString()); metadata.set(NFO.HAS_HASH, hash); metadata.set(NFO.FILE_SIZE, cos.getCount()); if (metadata.isNull(NFO.FILE_LAST_MODIFIED)) { metadata.set(NFO.FILE_LAST_MODIFIED, new Date()); resource.set(KS.STORED_AS, Record.create(metadata, true)); deleteFileQuietly(oldMetadata.getUnique(NFO.FILE_NAME, String.class));
@SuppressWarnings("unchecked") private Record decodeRecord(final GenericRecord generic, @Nullable final Set<URI> propertiesToDecode) { final Record record = Record.create(); final GenericRecord encodedID = (GenericRecord) generic.get(0); if (encodedID != null) { record.setID((URI) decodeIdentifier(encodedID)); } for (final GenericRecord prop : (Iterable<GenericRecord>) generic.get(1)) { final URI property = (URI) decodeIdentifier((GenericRecord) prop.get(0)); final List<Object> values = decodeNodes(prop.get(1)); if (propertiesToDecode == null || propertiesToDecode.contains(property)) { record.set(property, values); } } return record; }
@Override public Record computeNewRecord(final URI id, @Nullable final Record oldRecord, @Nullable final Record suppliedRecord) throws Throwable { assert suppliedRecord != null; if (criteria == null) { return oldRecord; // NOP } else { final Record record = oldRecord == null ? Record.create(id, type) : Record.create(oldRecord, true); criteria.merge(record, suppliedRecord); return record; } }
Stream<Record> recordStream = Record.decode(stmtStream, ImmutableList.of(type), true);
@Override public void handle(final Record record) throws Throwable { if (record == null || this.records > 0 && this.records % 1000 == 0) { LOGGER.info(this.records + " records, " + this.triples + " triples processed"); } if (record != null) { final List<Statement> statements = Record.encode(Stream.create(record), ImmutableSet.of()).toList(); writer.handleComment(record.getID().toString()); for (final Statement statement : statements) { writer.handleStatement(statement); } ++this.records; this.triples += statements.size(); } }
private synchronized void update(final Map<URI, Map<URI, Record>> tables, final int revision) throws IOException { if (this.revision != revision) { throw new IOException("Commit failed due to concurrent modifications " + this.revision + ", " + revision); } OutputStream stream = null; try { stream = Files.writeWithBackup(this.fileSystem, this.filePath); final List<Record> records = Lists.newArrayList(); for (final URI type : tables.keySet()) { records.addAll(tables.get(type).values()); } final RDFFormat format = RDFFormat.forFileName(this.filePath.getName()); RDFUtil.writeRDF(stream, format, Data.getNamespaceMap(), null, Record.encode(Stream.create(records), ImmutableSet.<URI>of())); ++this.revision; this.tables = tables; MemoryDataStore.LOGGER.info("MemoryDataStore updated, {} records persisted", records.size()); } catch (final Throwable ex) { MemoryDataStore.LOGGER.error("MemoryDataStore update failed", ex); } finally { Util.closeQuietly(stream); } }