public static List<NameComponentDescription> getRowComponents(TableMetadata tableMetadata) { NameMetadataDescription rowMetadata = tableMetadata.getRowMetadata(); List <NameComponentDescription> rowParts = rowMetadata.getRowParts(); return rowMetadata.numberOfComponentsHashed() == 0 ? rowParts : rowParts.subList(1, rowParts.size()); } }
private static Cell deserializeCell(TableMetadata metadata, JsonNode node) { byte[] row = deserializeRow(metadata.getRowMetadata(), node.get("row")); byte[] col = deserializeCol(metadata.getColumns(), node.get("col")); return Cell.create(row, col); }
@Override public TableRange deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { JsonNode node = jp.readValueAsTree(); String tableName = node.get("table").textValue(); TableMetadata metadata = metadataCache.getMetadata(tableName); JsonNode optBatchSize = node.get("batch_size"); int batchSize = optBatchSize == null ? 2000 : optBatchSize.asInt(); Iterable<byte[]> columns = AtlasDeserializers.deserializeNamedCols(metadata.getColumns(), node.get("cols")); byte[] startRow = new byte[0]; byte[] endRow = new byte[0]; if (node.has("prefix")) { startRow = AtlasDeserializers.deserializeRowPrefix(metadata.getRowMetadata(), node.get("prefix")); endRow = RangeRequests.createEndNameForPrefixScan(startRow); } else { if (node.has("raw_start")) { startRow = node.get("raw_start").binaryValue(); } else if (node.has("start")) { startRow = AtlasDeserializers.deserializeRow(metadata.getRowMetadata(), node.get("start")); } if (node.has("raw_end")) { endRow = node.get("raw_end").binaryValue(); } else if (node.has("end")) { endRow = AtlasDeserializers.deserializeRow(metadata.getRowMetadata(), node.get("end")); } } return new TableRange(tableName, startRow, endRow, columns, batchSize); } }
private MethodSpec renderNamedGetRangeStartEnd(NamedColumnDescription col) { Preconditions.checkArgument(tableMetadata.getRowMetadata().getRowParts().size() == 1); NameComponentDescription rowComponent = tableMetadata.getRowMetadata().getRowParts().get(0); MethodSpec.Builder getterBuilder = MethodSpec.methodBuilder("getSmallRowRange" + VarName(col)) .addModifiers(Modifier.PUBLIC) .addJavadoc("Returns a mapping from all the row keys in a range to their value at column $L\n" + "(if that column exists for the row-key). As the $L values are all loaded in memory,\n" + "do not use for large amounts of data. The order of results is preserved in the map.", VarName(col), VarName(col)) .addParameter(rowComponent.getType().getJavaClass(), "startInclusive") .addParameter(rowComponent.getType().getJavaClass(), "endExclusive") .returns(ParameterizedTypeName.get( ClassName.get(LinkedHashMap.class), ClassName.get(rowComponent.getType().getJavaClass()), ClassName.get(getColumnClassForGenericTypeParameter(col)))); getterBuilder .addStatement("$T rangeRequest = $T.builder()\n" + ".startRowInclusive($T.of(startInclusive).persistToBytes())\n" + ".endRowExclusive($T.of(endExclusive).persistToBytes())\n" + ".build()", RangeRequest.class, RangeRequest.class, rowType, rowType) .addStatement("return getSmallRowRange$L(rangeRequest)", VarName(col)); return getterBuilder.build(); }
@VisibleForTesting static void addLoggableNamesToBuilder( ImmutableSafeLoggableData.Builder builder, TableReference ref, TableMetadata tableMetadata) { if (IS_SAFE.test(tableMetadata.getNameLogSafety())) { builder.addPermittedTableReferences(ref); } // this is a system table with empty metadata, but safe for logging. builder.addPermittedTableReferences(AtlasDbConstants.DEFAULT_METADATA_TABLE); Set<String> loggableRowComponentNames = tableMetadata.getRowMetadata() .getRowParts() .stream() .filter(rowComponent -> IS_SAFE.test(rowComponent.getLogSafety())) .map(NameComponentDescription::getComponentName) .collect(Collectors.toSet()); builder.putPermittedRowComponents(ref, loggableRowComponentNames); Set<NamedColumnDescription> namedColumns = tableMetadata.getColumns().getNamedColumns(); if (namedColumns != null) { Set<String> loggableColumnNames = namedColumns .stream() .filter(columnComponent -> IS_SAFE.test(columnComponent.getLogSafety())) .map(NamedColumnDescription::getLongName) .collect(Collectors.toSet()); builder.putPermittedColumnNames(ref, loggableColumnNames); } } }
private List<RowNamePartitioner> getPartitioners(KeyValueService kvs, TableReference table) { try { byte[] metadata = kvs.getMetadataForTable(table); TableMetadata tableMeta = TableMetadata.BYTES_HYDRATOR.hydrateFromBytes(metadata); return tableMeta.getRowMetadata().getPartitionersForRow(); } catch (RuntimeException e) { processMessage( "Could not resolve partitioners from table metadata for " + table + " this may result in a small decrease in performance migrating this table.", e, KvsMigrationMessageLevel.WARN); return ImmutableList.of(); } }
@Override public TableRowSelection deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException { JsonNode node = jp.readValueAsTree(); String tableName = node.get("table").textValue(); TableMetadata metadata = metadataCache.getMetadata(tableName); Iterable<byte[]> rows = AtlasDeserializers.deserializeRows(metadata.getRowMetadata(), node.get("rows")); Iterable<byte[]> columns = AtlasDeserializers.deserializeNamedCols(metadata.getColumns(), node.get("cols")); if (Iterables.isEmpty(columns)) { return new TableRowSelection(tableName, rows, ColumnSelection.all()); } else { return new TableRowSelection(tableName, rows, ColumnSelection.create(columns)); } } }
private List<MethodSpec> renderNamedGet() { List<MethodSpec> getterResults = new ArrayList<>(); for (NamedColumnDescription col : ColumnRenderers.namedColumns(tableMetadata)) { getterResults.add(renderNamedGetColumn(col)); if (tableMetadata.getRowMetadata().getRowParts().size() == 1) { getterResults.add(renderNamedGetSeveralRows(col)); if (tableMetadata.isRangeScanAllowed()) { getterResults.add(renderNamedGetRangeColumn(col)); getterResults.add(renderNamedGetRangeStartEnd(col)); getterResults.add(renderNamedGetRangeColumnLimit(col)); } } else { getterResults.add(renderNamedGetSeveralRowObjects(col)); if (tableMetadata.isRangeScanAllowed()) { getterResults.add(renderNamedGetRangeColumnRowObjects(col)); getterResults.add(renderNamedGetRangeColumnRowObjectsLimit(col)); } } } return getterResults; }
@Override public void serialize(TableRowSelection value, JsonGenerator jgen, SerializerProvider provider) throws IOException { TableMetadata metadata = metadataCache.getMetadata(value.getTableName()); Preconditions.checkNotNull(metadata, "Unknown table %s", value.getTableName()); jgen.writeStartObject(); jgen.writeStringField("table", value.getTableName()); jgen.writeArrayFieldStart("rows"); for (byte[] row : value.getRows()) { AtlasSerializers.serializeRowish(jgen, metadata.getRowMetadata(), row); } jgen.writeEndArray(); if (!value.getColumnSelection().allColumnsSelected()) { jgen.writeArrayFieldStart("cols"); for (byte[] col : value.getColumnSelection().getSelectedColumns()) { jgen.writeUTF8String(col, 0, col.length); } jgen.writeEndArray(); } jgen.writeEndObject(); } }
private MethodSpec renderNamedGetSeveralRows(NamedColumnDescription col) { Preconditions.checkArgument(tableMetadata.getRowMetadata().getRowParts().size() == 1); NameComponentDescription rowComponent = tableMetadata.getRowMetadata().getRowParts().get(0); MethodSpec.Builder getterBuilder = MethodSpec.methodBuilder("get" + VarName(col)) .addModifiers(Modifier.PUBLIC)
private static Iterable<Entry<Cell, byte[]>> deserializeCellVal(TableMetadata metadata, JsonNode node) { byte[] row = deserializeRow(metadata.getRowMetadata(), node.get("row")); ColumnMetadataDescription colDescription = metadata.getColumns(); if (colDescription.hasDynamicColumns()) { byte[] col = deserializeDynamicCol(colDescription.getDynamicColumn(), node.get("col")); byte[] val = deserializeVal(colDescription.getDynamicColumn().getValue(), node.get("val")); return ImmutableList.of(Maps.immutableEntry(Cell.create(row, col), val)); } else { Collection<Entry<Cell, byte[]>> results = Lists.newArrayListWithCapacity(1); Iterator<Entry<String, JsonNode>> fields = node.fields(); while (fields.hasNext()) { Entry<String, JsonNode> entry = fields.next(); String longName = entry.getKey(); if (longName.equals("row")) { continue; } NamedColumnDescription description = getNamedCol(colDescription, longName); byte[] col = PtBytes.toCachedBytes(description.getShortName()); JsonNode valNode = entry.getValue(); byte[] val = deserializeVal(description.getValue(), valNode); results.add(Maps.immutableEntry(Cell.create(row, col), val)); } return results; } }
private MethodSpec renderNamedGetRangeColumn(NamedColumnDescription col) { Preconditions.checkArgument(tableMetadata.getRowMetadata().getRowParts().size() == 1); NameComponentDescription rowComponent = tableMetadata.getRowMetadata().getRowParts().get(0); MethodSpec.Builder getterBuilder = MethodSpec.methodBuilder("getSmallRowRange" + VarName(col)) .addModifiers(Modifier.PUBLIC)
private MethodSpec renderNamedGetRangeColumnLimit(NamedColumnDescription col) { Preconditions.checkArgument(tableMetadata.getRowMetadata().getRowParts().size() == 1); NameComponentDescription rowComponent = tableMetadata.getRowMetadata().getRowParts().get(0); MethodSpec.Builder getterBuilder = MethodSpec.methodBuilder("getSmallRowRange" + VarName(col)) .addModifiers(Modifier.PUBLIC)
private static void serialize(JsonGenerator jgen, TableMetadata metadata, Cell cell) throws IOException { byte[] row = cell.getRowName(); byte[] col = cell.getColumnName(); jgen.writeStartObject(); AtlasSerializers.serializeRow(jgen, metadata.getRowMetadata(), row); ColumnMetadataDescription columns = metadata.getColumns(); if (columns.hasDynamicColumns()) { DynamicColumnDescription dynamicColumn = columns.getDynamicColumn(); AtlasSerializers.serializeDynamicColumn(jgen, dynamicColumn, col); } else { String shortName = PtBytes.toString(col); jgen.writeStringField("col", getLongColumnName(columns, shortName)); } jgen.writeEndObject(); }
/** * Asserts that the only row component for the TableDefinition object passed in has loggability matching * expectedSafety. Throws if the actual safety doesn't match the expected safety, or if it is not the case that * there is exactly one row component. */ private static void assertRowComponentSafety(TableDefinition tableDefinition, LogSafety expectedSafety) { TableMetadata metadata = tableDefinition.toTableMetadata(); NameComponentDescription nameComponent = Iterables.getOnlyElement(metadata.getRowMetadata().getRowParts()); assertThat(nameComponent.getLogSafety()).isEqualTo(expectedSafety); }
RowResult<byte[]> result) throws IOException { jgen.writeStartObject(); AtlasSerializers.serializeRow(jgen, metadata.getRowMetadata(), result.getRowName()); ColumnMetadataDescription columns = metadata.getColumns(); if (columns.hasDynamicColumns()) {
private static void serialize(JsonGenerator jgen, TableMetadata metadata, Entry<Cell, byte[]> result) throws IOException { Cell cell = result.getKey(); byte[] row = cell.getRowName(); byte[] col = cell.getColumnName(); byte[] val = result.getValue(); jgen.writeStartObject(); AtlasSerializers.serializeRow(jgen, metadata.getRowMetadata(), row); ColumnMetadataDescription columns = metadata.getColumns(); if (columns.hasDynamicColumns()) { DynamicColumnDescription dynamicColumn = columns.getDynamicColumn(); AtlasSerializers.serializeDynamicColumn(jgen, dynamicColumn, col); jgen.writeFieldName("val"); AtlasSerializers.serializeVal(jgen, dynamicColumn.getValue(), val); } else { String shortName = PtBytes.toString(col); Set<NamedColumnDescription> namedColumns = columns.getNamedColumns(); for (NamedColumnDescription description : namedColumns) { if (shortName.equals(description.getShortName())) { AtlasSerializers.serializeNamedCol(jgen, description, val); break; } } } jgen.writeEndObject(); } }
@Override public void serialize(TableMetadata value, JsonGenerator jgen, SerializerProvider provider) throws IOException { jgen.writeStartObject(); NameMetadataDescription rowMetadata = value.getRowMetadata(); ColumnMetadataDescription columnMetadata = value.getColumns(); boolean isDynamic = columnMetadata.hasDynamicColumns();
TableMetadata metadata = metadataCache.getMetadata(tableName); for (JsonNode rowResult : node.get("data")) { byte[] row = AtlasDeserializers.deserializeRow(metadata.getRowMetadata(), rowResult.get("row")); ImmutableSortedMap.Builder<byte[], byte[]> cols = ImmutableSortedMap.orderedBy( UnsignedBytes.lexicographicalComparator());
public static String getHumanReadableCellName(@Nullable TableMetadata metadata, Cell cell) { if (cell == null) { return "null"; } if (metadata == null) { return cell.toString(); } try { String rowName = metadata.getRowMetadata().renderToJson(cell.getRowName()); String colName; if (metadata.getColumns().hasDynamicColumns()) { colName = metadata.getColumns().getDynamicColumn().getColumnNameDesc().renderToJson( cell.getColumnName()); } else { colName = PtBytes.toString(cell.getColumnName()); } return "Cell [rowName=" + rowName + ", columnName=" + colName + "]"; } catch (Exception e) { log.warn("Failed to render as json", e); return cell.toString(); } } }