/** * Copies the given bytes into a {@code ByteString}. */ public static ByteString copyFrom(final byte[] bytes) { return copyFrom(bytes, 0, bytes.length); }
/** * Read a {@code bytes} field value from the stream. */ @Override public ByteString readBytes() throws IOException { final int size = readRawVarint32(); if (size == 0) { return ByteString.EMPTY; } if (size <= (bufferSize - bufferPos) && size > 0) { // Fast path: We already have the bytes in a contiguous buffer, so // just copy directly from it. final ByteString result = ByteString.copyFrom(buffer, bufferPos, size); bufferPos += size; return result; } else { // Slow path: Build a byte array first then copy it. // return ByteString.copyFrom(readRawBytes(size)); return ByteString.wrap(readRawBytes(size)); } }
@Test public void testToString() throws Exception { byte[] array = new byte[] { 1, 2, 3, 4 }; ByteString s = ByteString.copyFrom(array); Assert.assertTrue(s.toString().contains("size=4")); }
public void testFooNullFieldsButFirst() throws Exception { Foo b = new Foo(); JsonIOUtil.mergeFrom(JsonIOUtil.DEFAULT_JSON_FACTORY.createJsonParser( "{\"someInt\":[1,null]" + ",\"someString\":[\"string\",null]" + ",\"someBar\":[{},null]" + ",\"someEnum\":[1,null]" + ",\"someBytes\":[\"fw==\",null]" + // 0x7f ",\"someBoolean\":[true,null]" + ",\"someFloat\":[10.01,null]" + ",\"someDouble\":[100.001,null]" + ",\"someLong\":[1000,null]}"), b, b.cachedSchema(), false); assertEquals(b.getSomeInt(), Arrays.asList(new Integer(1))); assertEquals(b.getSomeString(), Arrays.asList("string")); assertEquals(b.getSomeBar(), Arrays.asList(new Bar())); assertEquals(b.getSomeEnum(), Arrays.asList(Foo.EnumSample.TYPE1)); assertEquals(b.getSomeBytes(), Arrays.asList(ByteString.copyFrom(new byte[] { 0x7f }))); assertEquals(b.getSomeBoolean(), Arrays.asList(Boolean.TRUE)); assertEquals(b.getSomeFloat(), Arrays.asList(new Float(10.01f))); assertEquals(b.getSomeDouble(), Arrays.asList(new Double(100.001d))); assertEquals(b.getSomeLong(), Arrays.asList(new Long(1000l))); }
public void testFooNullFieldsButMid() throws Exception { Foo b = new Foo(); JsonIOUtil.mergeFrom(JsonIOUtil.DEFAULT_JSON_FACTORY.createJsonParser( "{\"someInt\":[null,1,null]" + ",\"someString\":[null,\"string\",null]" + ",\"someBar\":[null,{},null]" + ",\"someEnum\":[null,1,null]" + ",\"someBytes\":[null,\"fw==\",null]" + // 0x7f ",\"someBoolean\":[null,true,null]" + ",\"someFloat\":[null,10.01,null]" + ",\"someDouble\":[null,100.001,null]" + ",\"someLong\":[null,1000,null]}"), b, b.cachedSchema(), false); assertEquals(b.getSomeInt(), Arrays.asList(new Integer(1))); assertEquals(b.getSomeString(), Arrays.asList("string")); assertEquals(b.getSomeBar(), Arrays.asList(new Bar())); assertEquals(b.getSomeEnum(), Arrays.asList(Foo.EnumSample.TYPE1)); assertEquals(b.getSomeBytes(), Arrays.asList(ByteString.copyFrom(new byte[] { 0x7f }))); assertEquals(b.getSomeBoolean(), Arrays.asList(Boolean.TRUE)); assertEquals(b.getSomeFloat(), Arrays.asList(new Float(10.01f))); assertEquals(b.getSomeDouble(), Arrays.asList(new Double(100.001d))); assertEquals(b.getSomeLong(), Arrays.asList(new Long(1000l))); }
public void testFooNullFieldsButLast() throws Exception { Foo b = new Foo(); JsonIOUtil.mergeFrom(JsonIOUtil.DEFAULT_JSON_FACTORY.createJsonParser( "{\"someInt\":[null,1]" + ",\"someString\":[null,\"string\"]" + ",\"someBar\":[null,{}]" + ",\"someEnum\":[null,1]" + ",\"someBytes\":[null,\"fw==\"]" + // 0x7f ",\"someBoolean\":[null,true]" + ",\"someFloat\":[null,10.01]" + ",\"someDouble\":[null,100.001]" + ",\"someLong\":[null,1000]}"), b, b.cachedSchema(), false); assertEquals(b.getSomeInt(), Arrays.asList(new Integer(1))); assertEquals(b.getSomeString(), Arrays.asList("string")); assertEquals(b.getSomeBar(), Arrays.asList(new Bar())); assertEquals(b.getSomeEnum(), Arrays.asList(Foo.EnumSample.TYPE1)); assertEquals(b.getSomeBytes(), Arrays.asList(ByteString.copyFrom(new byte[] { 0x7f }))); assertEquals(b.getSomeBoolean(), Arrays.asList(Boolean.TRUE)); assertEquals(b.getSomeFloat(), Arrays.asList(new Float(10.01f))); assertEquals(b.getSomeDouble(), Arrays.asList(new Double(100.001d))); assertEquals(b.getSomeLong(), Arrays.asList(new Long(1000l))); }
public void testFooNullFieldsButLast2() throws Exception { Foo b = new Foo(); JsonIOUtil.mergeFrom(JsonIOUtil.DEFAULT_JSON_FACTORY.createJsonParser( "{\"someInt\":[null,null,1]" + ",\"someString\":[null,null,\"string\"]" + ",\"someBar\":[null,null,{}]" + ",\"someEnum\":[null,null,1]" + ",\"someBytes\":[null,null,\"fw==\"]" + // 0x7f ",\"someBoolean\":[null,null,true]" + ",\"someFloat\":[null,null,10.01]" + ",\"someDouble\":[null,null,100.001]" + ",\"someLong\":[null,null,1000]}"), b, b.cachedSchema(), false); assertEquals(b.getSomeInt(), Arrays.asList(new Integer(1))); assertEquals(b.getSomeString(), Arrays.asList("string")); assertEquals(b.getSomeBar(), Arrays.asList(new Bar())); assertEquals(b.getSomeEnum(), Arrays.asList(Foo.EnumSample.TYPE1)); assertEquals(b.getSomeBytes(), Arrays.asList(ByteString.copyFrom(new byte[] { 0x7f }))); assertEquals(b.getSomeBoolean(), Arrays.asList(Boolean.TRUE)); assertEquals(b.getSomeFloat(), Arrays.asList(new Float(10.01f))); assertEquals(b.getSomeDouble(), Arrays.asList(new Double(100.001d))); assertEquals(b.getSomeLong(), Arrays.asList(new Long(1000l))); }
@Override public ByteString deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException { return ByteString.copyFrom(p.getBinaryValue()); }
@Override public ByteString toBytesString() { return ByteString.copyFrom(toBytes()); }
public ByteString readBytes() throws IOException { return ByteString.copyFrom(this.readByteArray()); }
public ByteString toByteString(){ return ByteString.copyFrom(serialize()); }
private static ByteString convertSearchConfigurationToByteString(SearchConfiguration configuration) { final LinkedBuffer buffer = LinkedBuffer.allocate(); byte[] bytes = ProtostuffIOUtil.toByteArray(configuration, SearchConfiguration.getSchema(), buffer); return ByteString.copyFrom(bytes); } }
private static ByteString convertClusterIdentityToByteString(ClusterIdentity identity) { final LinkedBuffer buffer = LinkedBuffer.allocate(); byte[] bytes = ProtostuffIOUtil.toByteArray(identity, ClusterIdentity.getSchema(), buffer); return ByteString.copyFrom(bytes); }
@Override public DatasetConfig updateForSchemaChange(DatasetConfig oldConfig, BatchSchema expectedSchema, BatchSchema newlyObservedSchema) { Preconditions.checkNotNull(oldConfig); Preconditions.checkNotNull(newlyObservedSchema); BatchSchema newSchema; if (DatasetHelper.getSchemaBytes(oldConfig) == null) { newSchema = newlyObservedSchema; } else { newSchema = BatchSchema.fromDataset(oldConfig).merge(newlyObservedSchema); } DatasetConfig newConfig = clone(oldConfig); newConfig.setRecordSchema(ByteString.copyFrom(newSchema.serialize())); return newConfig; }
@Override public void recordExtraInfo(String name, byte[] bytes) { //TODO DX-10977 the reflection manager should rely on its own observer to store this information in a separate store if(job.getJobAttempt().getExtraInfoList() == null) { job.getJobAttempt().setExtraInfoList(new ArrayList<ExtraInfo>()); } job.getJobAttempt().getExtraInfoList().add(new ExtraInfo() .setData(ByteString.copyFrom(bytes)) .setName(name)); storeJob(job); super.recordExtraInfo(name, bytes); }
private void buildAll(DatasetConfig datasetConfig) throws Exception { final EasyGroupScanUtils easyGroupScanUtils = ((EasyFormatPlugin) formatPlugin).getGroupScan(SYSTEM_USERNAME, fsPlugin, fileSelection, GroupScan.ALL_COLUMNS); cachedMetadata = new ReadDefinition() .setLastRefreshDate(System.currentTimeMillis()) .setScanStats(MetadataUtils.fromPojoScanStats(easyGroupScanUtils.getScanStats()).setScanFactor(ScanCostFactor.EASY.getFactor())) .setReadSignature(ByteString.copyFrom(FILE_UPDATE_KEY_SERIALIZER.serialize(updateKey))) .setPartitionColumnsList(MetadataUtils.getStringColumnNames(easyGroupScanUtils.getPartitionColumns())) .setExtendedProperty( ByteString.copyFrom(EasyDatasetXAttrSerDe.EASY_DATASET_XATTR_SERIALIZER.serialize( new EasyDatasetXAttr().setSelectionRoot(fileSelection.getSelectionRoot())))); //cachedMetadata.setSortColumnsList(easyGroupScanUtils.getSortColumns()); // TODO(AH) probably not needed since they are set in layout info? // compute splits this.cachedSplits = getSplits(datasetConfig, easyGroupScanUtils); this.builtAll = true; }
private Stream<DatasetSplit> generateSplits(long splitVersion, int count) { return IntStream.range(0, count) .mapToObj(i -> new DatasetSplit() .setRowCount((long) i) .setTag("0") .setSize((long) i) .setAffinitiesList(Arrays.asList(new Affinity().setHost("node" + i))) .setPartitionValuesList(Lists.newArrayList(new PartitionValue().setColumn("column" + i).setIntValue(i).setType(PartitionValueType.IMPLICIT))) .setExtendedProperty(ByteString.copyFrom(String.valueOf(i).getBytes())) .setSplitKey(String.valueOf(i)) .setSplitVersion(splitVersion) ); }
private ByteString convertToScanXAttr(ByteString xattrFullSerialized) { ParquetDatasetSplitXAttr fullXAttr = ParquetDatasetXAttrSerDe.PARQUET_DATASET_SPLIT_XATTR_SERIALIZER.revert(xattrFullSerialized.toByteArray());; ParquetDatasetSplitScanXAttr scanXAttr = new ParquetDatasetSplitScanXAttr(); scanXAttr.setPath(fullXAttr.getPath()); scanXAttr.setFileLength(fullXAttr.getUpdateKey().getLength()); scanXAttr.setStart(fullXAttr.getStart()); scanXAttr.setLength(fullXAttr.getLength()); scanXAttr.setRowGroupIndex(fullXAttr.getRowGroupIndex()); return ByteString.copyFrom(ParquetDatasetXAttrSerDe.PARQUET_DATASET_SPLIT_SCAN_XATTR_SERIALIZER.serialize(scanXAttr)); }
public static void addPhysicalDS(NamespaceService ns, String filePath, DatasetType type, byte[] datasetSchema) throws Exception { NamespaceKey datasetPath = new NamespaceKey(PathUtils.parseFullPath(filePath)); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); final PhysicalDataset physicalDataset = new PhysicalDataset(); if (datasetSchema != null) { datasetConfig.setRecordSchema(io.protostuff.ByteString.copyFrom(datasetSchema)); } datasetConfig.setSchemaVersion(DatasetHelper.CURRENT_VERSION); datasetConfig.setPhysicalDataset(physicalDataset); ns.tryCreatePhysicalDataset(datasetPath, datasetConfig); }
@Test public void testStore() throws Exception { try(final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false)) { kvstore.start(); ConfigurationStore store = new ConfigurationStore(kvstore); ConfigurationEntry supportEntry = new ConfigurationEntry(); supportEntry.setType("mytype"); supportEntry.setValue(ByteString.copyFrom("test string", "UTF8")); store.put("key", supportEntry); ConfigurationEntry retrieved = store.get("key"); Assert.assertEquals(retrieved.getType(), supportEntry.getType()); Assert.assertEquals(retrieved.getValue(), supportEntry.getValue()); } } }