char marker = (char) splitter.getInt(); LOG.trace("Got marker {}", marker); EntityId id1 = toEntityId(splitter, marker); rowInfo.put(marker, id1); splitter.skipLong(); // inverted time - not required for relation marker = (char) splitter.getInt(); LOG.trace("Got marker {}", marker); EntityId id2 = toEntityId(splitter, marker); rowInfo.put(marker, id2); RunId runId = RunIds.fromString(splitter.getString()); LOG.trace("Got runId {}", runId); AccessType accessType = AccessType.fromType((char) splitter.getInt()); LOG.trace("Got access type {}", accessType);
public List<ApplicationId> getAllAppVersionsAppIds(String namespaceId, String appId) { List<ApplicationId> appIds = new ArrayList<>(); for (MDSKey key : listKV(new MDSKey.Builder().add(TYPE_APP_META, namespaceId, appId).build(), ApplicationMeta.class).keySet()) { MDSKey.Splitter splitter = key.split(); splitter.skipBytes(); // skip recordType splitter.skipBytes(); // skip namespaceId splitter.skipBytes(); // skip appId String versionId = splitter.hasRemaining() ? splitter.getString() : ApplicationId.DEFAULT_VERSION; appIds.add(new NamespaceId(namespaceId).app(appId, versionId)); } return appIds; }
@Override public Multimap<NamespaceId, StreamSpecification> call(DatasetContext context) throws Exception { ImmutableMultimap.Builder<NamespaceId, StreamSpecification> builder = ImmutableMultimap.builder(); Map<MDSKey, StreamSpecification> streamSpecs = getMetadataStore(context).listKV(new MDSKey.Builder().add(TYPE_STREAM).build(), StreamSpecification.class); for (Map.Entry<MDSKey, StreamSpecification> streamSpecEntry : streamSpecs.entrySet()) { MDSKey.Splitter splitter = streamSpecEntry.getKey().split(); // skip the first name ("stream") splitter.skipString(); // Namespace id is the next part. String namespaceId = splitter.getString(); builder.put(new NamespaceId(namespaceId), streamSpecEntry.getValue()); } return builder.build(); } }, Exception.class);
public static NamespacedEntityId getTargetIdIdFromKey(MDSKey.Splitter keySplitter, String type) { type = type.toLowerCase(); if (type.equals(TYPE_MAP.get(NamespaceId.class))) { String namespaceId = keySplitter.getString(); return new NamespaceId(namespaceId); } else if (type.equals(TYPE_MAP.get(ProgramId.class))) { String namespaceId = keySplitter.getString(); String appId = keySplitter.getString(); String programType = keySplitter.getString(); String programId = keySplitter.getString(); return new ProgramId(namespaceId, appId, programType, programId); } else if (type.equals(TYPE_MAP.get(ApplicationId.class))) { String namespaceId = keySplitter.getString(); String appId = keySplitter.getString(); return new ApplicationId(namespaceId, appId); } else if (type.equals(TYPE_MAP.get(ArtifactId.class))) { String namespaceId = keySplitter.getString(); String name = keySplitter.getString(); String version = keySplitter.getString(); return new ArtifactId(namespaceId, name, version); } else if (type.equals(TYPE_MAP.get(DatasetId.class)) || type.equals("datasetinstance")) { String namespaceId = keySplitter.getString(); String instanceId = keySplitter.getString(); return new DatasetId(namespaceId, instanceId); } else if (type.equals(TYPE_MAP.get(StreamId.class))) { String namespaceId = keySplitter.getString(); String instanceId = keySplitter.getString(); return new StreamId(namespaceId, instanceId); } else if (type.equals(TYPE_MAP.get(StreamViewId.class)) || type.equals("view")) { String namespaceId = keySplitter.getString();
MDSKey.Splitter keySplitter = new MDSKey(rowKey).split(); keySplitter.skipBytes(); keySplitter.skipString(); keySplitter.skipString(); keySplitter.skipString(); keySplitter.skipString(); break; case APPLICATION: case "datasetinstance": case STREAM: keySplitter.skipString(); keySplitter.skipString(); break; case VIEW: case "view": case ARTIFACT: keySplitter.skipString(); keySplitter.skipString(); keySplitter.skipString(); break; default: return Long.MAX_VALUE - keySplitter.getLong();
keySplitter.skipBytes(); keySplitter.skipString(); type = type.toLowerCase(); switch (type) { case PROGRAM: keySplitter.skipString(); keySplitter.skipString(); keySplitter.skipString(); keySplitter.skipString(); break; case APPLICATION: case "datasetinstance": case STREAM: keySplitter.skipString(); keySplitter.skipString(); break; case VIEW: case "view": case ARTIFACT: keySplitter.skipString(); keySplitter.skipString(); keySplitter.skipString(); break; default: return keySplitter.getString();
MDSKey.Splitter splitter = new MDSKey(row.getRow()).split(); splitter.getString(); for (String key : tableSchema.getPrimaryKeys()) { switch (Objects.requireNonNull(type)) { case INTEGER: int intVal = splitter.getInt(); builder.put(key, intVal); keys.add(Fields.intField(key, intVal)); break; case LONG: long longVal = splitter.getLong(); builder.put(key, longVal); keys.add(Fields.longField(key, longVal)); break; case STRING: String stringVal = splitter.getString(); keys.add(Fields.stringField(key, stringVal)); builder.put(key, stringVal); break; case BYTES: byte[] bytesVal = splitter.getBytes(); keys.add(Fields.bytesField(key, bytesVal)); builder.put(key, bytesVal);
private static MetadataEntity getTargetIdIdFromKey(MDSKey.Splitter keySplitter) { String targetType = keySplitter.getString(); String key = keySplitter.getString(); String value = keySplitter.getString(); MetadataEntity.Builder builder = MetadataEntity.builder(); while (keySplitter.hasRemaining()) { builder = builder.append(key, value); key = keySplitter.getString(); if (keySplitter.hasRemaining()) { value = keySplitter.getString(); } else { break;
private Relation toRelation(Row row) { Map<Character, EntityId> rowInfo = new HashMap<>(4); MDSKey.Splitter splitter = new MDSKey(row.getRow()).split(); char marker = (char) splitter.getInt(); LOG.trace("Got marker {}", marker); EntityId id1 = toEntityId(splitter, marker); LOG.trace("Got id1 {}", id1); rowInfo.put(marker, id1); splitter.skipLong(); // inverted time - not required for relation marker = (char) splitter.getInt(); LOG.trace("Got marker {}", marker); EntityId id2 = toEntityId(splitter, marker); LOG.trace("Got id2 {}", id1); rowInfo.put(marker, id2); RunId runId = RunIds.fromString(splitter.getString()); LOG.trace("Got runId {}", runId); AccessType accessType = AccessType.fromType((char) splitter.getInt()); LOG.trace("Got access type {}", accessType); DatasetId datasetInstance = (DatasetId) rowInfo.get(DATASET_MARKER); LOG.trace("Got datasetInstance {}", datasetInstance); ProgramId program = (ProgramId) rowInfo.get(PROGRAM_MARKER); LOG.trace("Got program {}", program); NamespacedEntityId component = toComponent(splitter, program); LOG.trace("Got component {}", component); return new Relation(datasetInstance, program, accessType, runId, component == null ? Collections.emptySet() : Collections.singleton(component)); }
private static MetadataEntity getTargetIdIdFromKey(MDSKey.Splitter keySplitter) { String targetType = keySplitter.getString(); String key = keySplitter.getString(); String value = keySplitter.getString(); MetadataEntity.Builder builder = MetadataEntity.builder(); while (keySplitter.hasRemaining()) { builder = builder.append(key, value); key = keySplitter.getString(); if (keySplitter.hasRemaining()) { value = keySplitter.getString(); } else { break;
private RowKey parseRow(Row row) { ProgramId program; NamespacedEntityId data; RunId runId; MDSKey.Splitter splitter = new MDSKey(row.getRow()).split(); char marker = (char) splitter.getInt(); LOG.trace("Got marker {}", marker); switch (marker) { case PROGRAM_MARKER: program = (ProgramId) toEntityId(splitter, marker); splitter.skipLong(); // inverted start time marker = (char) splitter.getInt(); data = toEntityId(splitter, marker); // data runId = RunIds.fromString(splitter.getString()); return new RowKey(program, data, runId); case DATASET_MARKER: case STREAM_MARKER: data = toEntityId(splitter, marker); splitter.skipLong(); // inverted start time marker = (char) splitter.getInt(); program = (ProgramId) toEntityId(splitter, marker); // program runId = RunIds.fromString(splitter.getString()); return new RowKey(program, data, runId); default: throw new IllegalStateException("Invalid row with marker " + marker); } }
private RowKey parseRow(Row row) { ProgramId program; NamespacedEntityId data; RunId runId; MDSKey.Splitter splitter = new MDSKey(row.getRow()).split(); char marker = (char) splitter.getInt(); LOG.trace("Got marker {}", marker); switch (marker) { case PROGRAM_MARKER: program = (ProgramId) toEntityId(splitter, marker); splitter.skipLong(); // inverted start time marker = (char) splitter.getInt(); data = toEntityId(splitter, marker); // data runId = RunIds.fromString(splitter.getString()); return new RowKey(program, data, runId); case DATASET_MARKER: data = toEntityId(splitter, marker); splitter.skipLong(); // inverted start time marker = (char) splitter.getInt(); program = (ProgramId) toEntityId(splitter, marker); // program runId = RunIds.fromString(splitter.getString()); return new RowKey(program, data, runId); default: throw new IllegalStateException("Invalid row with marker " + marker); } }
static String extractMetadataKey(byte[] rowKey) { MDSKey.Splitter keySplitter = new MDSKey(rowKey).split(); // The rowkey is // [rowPrefix][targetType][targetId][key] for value rows and // [rowPrefix][targetType][targetId][key][index] for value index rows // Skip rowPrefix keySplitter.skipBytes(); // Skip targetType keySplitter.skipString(); // targetId are key-value par so always in set of two. For value row we will end up with only string in end ([key]) // and for index row we will have two strings in end ([key][index]). String key = null; while (keySplitter.hasRemaining()) { key = keySplitter.getString(); if (keySplitter.hasRemaining()) { keySplitter.skipString(); } else { break; } } return key; }
static String extractMetadataKey(byte[] rowKey) { MDSKey.Splitter keySplitter = new MDSKey(rowKey).split(); // The rowkey is // [rowPrefix][targetType][targetId][key] for value rows and // [rowPrefix][targetType][targetId][key][index] for value index rows // Skip rowPrefix keySplitter.skipBytes(); // Skip targetType keySplitter.skipString(); // targetId are key-value par so always in set of two. For value row we will end up with only string in end ([key]) // and for index row we will have two strings in end ([key][index]). String key = null; while (keySplitter.hasRemaining()) { key = keySplitter.getString(); if (keySplitter.hasRemaining()) { keySplitter.skipString(); } else { break; } } return key; }
@Test public void testGetBytesOverflow() { MDSKey.Builder builder = new MDSKey.Builder(); builder.add(2000); builder.add(2000); MDSKey mdsKey = builder.build(); MDSKey.Splitter splitter = mdsKey.split(); // splitter.getBytes and splitter.getString() will fail due to the key being composed of two large int parts try { splitter.getBytes(); Assert.fail(); } catch (BufferUnderflowException expected) { } try { splitter.getString(); Assert.fail(); } catch (BufferUnderflowException expected) { } }
@Test public void testSkipStringAndBytes() { MDSKey.Builder builder = new MDSKey.Builder(); builder.add("part1"); builder.add("part2"); builder.add("part3"); byte[] bytesToSkip = new byte[] { 0x1 }; byte[] bytesToCheck = new byte[] { 0x2 }; builder.add(bytesToSkip); builder.add(bytesToCheck); MDSKey mdsKey = builder.build(); MDSKey.Splitter splitter = mdsKey.split(); Assert.assertEquals("part1", splitter.getString()); splitter.skipString(); Assert.assertEquals("part3", splitter.getString()); splitter.skipBytes(); Assert.assertTrue(splitter.hasRemaining()); Assert.assertTrue(Bytes.equals(bytesToCheck, splitter.getBytes())); Assert.assertFalse(splitter.hasRemaining()); }
/** * Returns a ProgramId given the MDS key * * @param key the MDS key to be used * @return ProgramId created from the MDS key */ private static ProgramId getProgramID(MDSKey key) { MDSKey.Splitter splitter = key.split(); // Format : recordType, ns, app, version, type, program, ts, runid // record type splitter.getString(); String namespace = splitter.getString(); String application = splitter.getString(); String appVersion = splitter.getString(); String type = splitter.getString(); String program = splitter.getString(); return (new ApplicationId(namespace, application, appVersion).program(ProgramType.valueOf(type), program)); }
/** * Returns a ProgramId given the MDS key * * @param key the MDS key to be used * @return ProgramId created from the MDS key */ private static ProgramId getProgramID(MDSKey key) { MDSKey.Splitter splitter = key.split(); // Format : recordType, ns, app, version, type, program, ts, runid // record type splitter.getString(); String namespace = splitter.getString(); String application = splitter.getString(); String appVersion = splitter.getString(); String type = splitter.getString(); String program = splitter.getString(); return (new ApplicationId(namespace, application, appVersion).program(ProgramType.valueOf(type), program)); }
@Test public void testGetMDSIndexKey() { MDSKey mdsIndexKey = MetadataKey.createIndexRowKey(new ApplicationId("ns1", "app1").toMetadataEntity(), "key1", "value1"); MDSKey.Splitter split = mdsIndexKey.split(); // skip value key bytes split.skipBytes(); // assert target type Assert.assertEquals(MetadataEntity.APPLICATION, split.getString()); // assert key-value pairs Assert.assertEquals(MetadataEntity.NAMESPACE, split.getString()); Assert.assertEquals("ns1", split.getString()); Assert.assertEquals(MetadataEntity.APPLICATION, split.getString()); Assert.assertEquals("app1", split.getString()); Assert.assertEquals("key1", split.getString()); Assert.assertEquals("value1", split.getString()); Assert.assertFalse(split.hasRemaining()); } }
@Test public void getGetIntOverflow() { MDSKey.Builder builder = new MDSKey.Builder(); builder.add(1); builder.add(2); builder.add(3); MDSKey mdsKey = builder.build(); MDSKey.Splitter splitter = mdsKey.split(); Assert.assertEquals(1, splitter.getInt()); Assert.assertEquals(2, splitter.getInt()); Assert.assertEquals(3, splitter.getInt()); // splitter.getInt will fail due to there only being 3 parts in the key try { splitter.getInt(); Assert.fail(); } catch (BufferUnderflowException expected) { Assert.assertFalse(splitter.hasRemaining()); } }