/** * Deletes a row, given a Scanner of JUST that row. */ private void deleteRow(Scanner scanner, BatchWriter bw) throws MutationsRejectedException { Mutation deleter = null; // iterate through the keys final Text row = new Text(); final Text cf = new Text(); final Text cq = new Text(); for (Entry<Key, Value> entry : scanner) { // create a mutation for the row if (deleter == null) { entry.getKey().getRow(row); deleter = new Mutation(row); } entry.getKey().getColumnFamily(cf); entry.getKey().getColumnQualifier(cq); // the remove function adds the key with the delete flag set to true deleter.putDelete(cf, cq); } bw.addMutation(deleter); } }
/** * Flushes all Mutations in the index writer. And all metric mutations to the metrics table. * Note that the metrics table is not updated until this method is explicitly called (or implicitly via close). */ public void flush() { try { // Flush index writer indexWriter.flush(); // Write out metrics mutations BatchWriter metricsWriter = connector.createBatchWriter(table.getMetricsTableName(), writerConfig); metricsWriter.addMutations(getMetricsMutations()); metricsWriter.close(); // Re-initialize the metrics metrics.clear(); metrics.put(METRICS_TABLE_ROW_COUNT, new AtomicLong(0)); } catch (MutationsRejectedException e) { throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Index mutation was rejected by server on flush", e); } catch (TableNotFoundException e) { throw new PrestoException(ACCUMULO_TABLE_DNE, "Accumulo table does not exist", e); } }
@Override public CompletableFuture<Collection<Slice>> finish() { try { // Done serializing rows, so flush and close the writer and indexer writer.flush(); writer.close(); if (indexer.isPresent()) { indexer.get().close(); } } catch (MutationsRejectedException e) { throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Mutation rejected by server on flush", e); } // TODO Look into any use of the metadata for writing out the rows return completedFuture(ImmutableList.of()); }
private void writeTestMutation(Connector userC) throws TableNotFoundException, MutationsRejectedException { BatchWriter batchWriter = userC.createBatchWriter(tableName, new BatchWriterConfig()); Mutation m = new Mutation("1"); m.put(new Text("2"), new Text("3"), new Value("".getBytes())); batchWriter.addMutation(m); batchWriter.flush(); batchWriter.close(); }
VolumeManager fs = master.getFileSystem(); mbw = master.getContext().createBatchWriter(MetadataTable.NAME, new BatchWriterConfig()); Value val = new Value(); val.readFields(in); if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { String oldName = new Path(key.getColumnQualifier().toString()).getName(); String newName = fileNameMappings.get(oldName); cq = new Text(bulkDir + "/" + newName); } else { cq = key.getColumnQualifier(); if (!currentRow.equals(metadataRow)) { mbw.addMutation(m); m.put(key.getColumnFamily(), cq, val); mbw.addMutation(m); break; // its the last column in the last row mbw.close();
@Test public void testCloneWithSplits() throws Exception { Connector conn = getConnector(); List<Mutation> mutations = new ArrayList<>(); TreeSet<Text> splits = new TreeSet<>(); for (int i = 0; i < 10; i++) { splits.add(new Text(Integer.toString(i))); Mutation m = new Mutation(Integer.toString(i)); m.put("", "", ""); mutations.add(m); } String[] tables = getUniqueNames(2); conn.tableOperations().create(tables[0]); conn.tableOperations().addSplits(tables[0], splits); BatchWriter bw = conn.createBatchWriter(tables[0], new BatchWriterConfig()); bw.addMutations(mutations); bw.close(); conn.tableOperations().clone(tables[0], tables[1], true, null, null); conn.tableOperations().deleteRows(tables[1], new Text("4"), new Text("8")); List<String> rows = Arrays.asList("0", "1", "2", "3", "4", "9"); List<String> actualRows = new ArrayList<>(); for (Entry<Key,Value> entry : conn.createScanner(tables[1], Authorizations.EMPTY)) { actualRows.add(entry.getKey().getRow().toString()); } Assert.assertEquals(rows, actualRows); }
public void writeAndReadData(Connector connector, String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { // Write some data to the table BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig()); for (String s : rows) { Mutation m = new Mutation(new Text(s)); m.put(EMPTY, EMPTY, EMPTY_VALUE); bw.addMutation(m); } bw.close(); // Write the data to disk, read it back connector.tableOperations().flush(tableName, null, null, true); Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY); int i = 0; for (Entry<Key,Value> entry : scanner) { assertEquals("Data read is not data written", rows[i++], entry.getKey().getRow().toString()); } }
Mutation m = new Mutation(row); Map<Table.ID,Long> tableToTimeCreated = new HashMap<>(); for (Entry<Key,Value> entry : columns.entrySet()) { Status status = null; try { status = Status.parseFrom(entry.getValue().get()); } catch (InvalidProtocolBufferException e) { log.error("Encountered unparsable protobuf for key: {}", m.putDelete(colf, colq); if (StatusSection.NAME.equals(colf)) { tableId = Table.ID.of(colq.toString()); } else if (WorkSection.NAME.equals(colf)) { ReplicationTarget target = ReplicationTarget.from(colq); tableId = target.getSourceTableId(); entry.getValue(), row); Mutation orderMutation = OrderSection.createMutation(row.toString(), entry.getValue()); orderMutation.putDelete(OrderSection.NAME, new Text(entry.getKey().getUtf8())); mutations.add(orderMutation); bw.addMutations(mutations); bw.flush(); } catch (MutationsRejectedException e) { log.error("Could not submit mutation to remove columns for {} in replication table", row, e);
long recordsClosed = 0; try { bw = client.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig()); bs = client.createBatchScanner(MetadataTable.NAME, Authorizations.EMPTY, 4); bs.setRanges(Collections.singleton(Range.prefix(ReplicationSection.getRowPrefix()))); bs.fetchColumnFamily(ReplicationSection.COLF); Text replFileText = new Text(); for (Entry<Key,Value> entry : bs) { Status status; try { status = Status.parseFrom(entry.getValue().get()); } catch (InvalidProtocolBufferException e) { log.error("Could not parse Status protobuf for {}", entry.getKey(), e); String replFile = replFileText.toString(); boolean isClosed = closedWals.contains(replFile); bw.close(); } catch (MutationsRejectedException e) { log.error("Failed to write delete mutations for replication table", e);
TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner); scanner.fetchColumnFamily(DataFileColumnFamily.NAME); Mutation m = new Mutation(stopRow); String maxLogicalTime = null; for (Entry<Key,Value> entry : scanner) { Key key = entry.getKey(); Value value = entry.getValue(); if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { m.put(key.getColumnFamily(), key.getColumnQualifier(), value); fileCount++; } else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key) && firstPrevRowValue == null) { Master.log.debug("prevRow entry for lowest tablet is {}", value); firstPrevRowValue = new Value(value); } else if (TabletsSection.ServerColumnFamily.TIME_COLUMN.hasColumns(key)) { maxLogicalTime = TabletTime.maxMetadataTime(maxLogicalTime, value.toString()); } else if (TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) { bw.addMutation(MetadataTableUtil.createDeleteMutation(master.getContext(), range.getTableId(), entry.getValue().toString())); if (!m.getUpdates().isEmpty()) { bw.addMutation(m); bw.flush(); Mutation updatePrevRow = stop.getPrevRowUpdateMutation(); Master.log.debug("Setting the prevRow for last tablet: {}", stop); bw.addMutation(updatePrevRow); bw.flush();
/** * Create a status record in the replication table */ protected boolean addStatusRecord(Text file, Table.ID tableId, Value v) { try { Mutation m = new Mutation(file); m.put(StatusSection.NAME, new Text(tableId.getUtf8()), v); try { replicationWriter.addMutation(m); } catch (MutationsRejectedException e) { log.warn("Failed to write work mutations for replication, will retry", e); return false; } } finally { try { replicationWriter.flush(); } catch (MutationsRejectedException e) { log.warn("Failed to write work mutations for replication, will retry", e); return false; } } return true; }
try (Scanner ms = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY); BatchWriter bw = new BatchWriterImpl(context, MetadataTable.ID, new BatchWriterConfig().setMaxMemory(1000000) .setMaxLatency(120000L, TimeUnit.MILLISECONDS).setMaxWriteThreads(2))) { Key key = cell.getKey(); if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { FileRef ref = new FileRef(context.getVolumeManager(), key); bw.addMutation(createDeleteMutation(context, tableId, ref.meta().toString())); bw.addMutation(createDeleteMutation(context, tableId, cell.getValue().toString())); bw.flush(); m = new Mutation(key.getRow()); if (lock != null) putLockID(context, lock, m); if (key.getRow().compareTo(m.getRow(), 0, m.getRow().length) != 0) { bw.addMutation(m); m = new Mutation(key.getRow()); if (lock != null) bw.addMutation(m);
shellState.checkTableState(); final Mutation m = new Mutation(new Text(cl.getArgs()[0].getBytes(Shell.CHARSET))); final Text colf = new Text(cl.getArgs()[1].getBytes(Shell.CHARSET)); final Text colq = new Text(cl.getArgs()[2].getBytes(Shell.CHARSET)); final Value val = new Value(cl.getArgs()[3].getBytes(Shell.CHARSET)); m.put(colf, colq, le, Long.parseLong(cl.getOptionValue(timestampOpt.getOpt())), val); else m.put(colf, colq, le, val); } else if (cl.hasOption(timestampOpt.getOpt())) m.put(colf, colq, Long.parseLong(cl.getOptionValue(timestampOpt.getOpt())), val); m.put(colf, colq, val); final BatchWriterConfig cfg = new BatchWriterConfig() .setMaxMemory(Math.max(m.estimatedMemoryUsed(), 1024)).setMaxWriteThreads(1) .setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS); if (cl.hasOption(durabilityOption.getOpt())) { bw.addMutation(m); try { bw.close(); } catch (MutationsRejectedException e) { final ArrayList<String> lines = new ArrayList<>();
private void makeFile(Connector conn, String tablename) throws Exception { BatchWriter bw = conn.createBatchWriter(tablename, new BatchWriterConfig()); byte[] empty = {}; byte[] row = new byte[10]; r.nextBytes(row); Mutation m = new Mutation(row, 0, 10); m.put(empty, empty, empty); bw.addMutation(m); bw.flush(); bw.close(); conn.tableOperations().flush(tablename, null, null, true); }
private void addDuplicateLocation(String table, String tableNameToModify) throws TableNotFoundException, MutationsRejectedException { String tableIdToModify = getConnector().tableOperations().tableIdMap().get(tableNameToModify); Mutation m = new Mutation(new KeyExtent(tableIdToModify, null, null).getMetadataEntry()); m.put(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME, new Text("1234567"), new Value("fake:9005".getBytes(UTF_8))); BatchWriter bw = getConnector().createBatchWriter(table, null); bw.addMutation(m); bw.close(); }
log.info("Adding work records for {} to targets {}", file, targets); try { Mutation m = new Mutation(file); Text t = new Text(); for (Entry<String,String> entry : targets.entrySet()) { buffer.reset(); t.set(buffer.getData(), 0, buffer.getLength()); writer.addMutation(m); } catch (MutationsRejectedException e) { log.warn("Failed to write work mutations for replication, will retry", e); } finally { try { writer.flush(); } catch (MutationsRejectedException e) { log.warn("Failed to write work mutations for replication, will retry", e);
public static void removeBulkLoadEntries(Connector conn, String tableId, long tid) throws Exception { Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS)); mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange()); mscanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY); BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig()); for (Entry<Key,Value> entry : mscanner) { log.debug("Looking at entry " + entry + " with tid " + tid); if (Long.parseLong(entry.getValue().toString()) == tid) { log.debug("deleting entry " + entry); Mutation m = new Mutation(entry.getKey().getRow()); m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier()); bw.addMutation(m); } } bw.close(); }
@Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException, ConstraintViolationException { shellState.checkTableState(); final Mutation m = new Mutation(new Text(cl.getArgs()[0].getBytes(Shell.CHARSET))); final Text colf = new Text(cl.getArgs()[1].getBytes(Shell.CHARSET)); final Text colq = new Text(cl.getArgs()[2].getBytes(Shell.CHARSET)); if (cl.hasOption(deleteOptAuths.getOpt())) { final ColumnVisibility le = new ColumnVisibility(cl.getOptionValue(deleteOptAuths.getOpt())); if (cl.hasOption(timestampOpt.getOpt())) { m.putDelete(colf, colq, le, Long.parseLong(cl.getOptionValue(timestampOpt.getOpt()))); } else { m.putDelete(colf, colq, le); } } else if (cl.hasOption(timestampOpt.getOpt())) { m.putDelete(colf, colq, Long.parseLong(cl.getOptionValue(timestampOpt.getOpt()))); } else { m.putDelete(colf, colq); } final BatchWriter bw = shellState.getAccumuloClient().createBatchWriter( shellState.getTableName(), new BatchWriterConfig().setMaxMemory(Math.max(m.estimatedMemoryUsed(), 1024)) .setMaxWriteThreads(1).setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS)); bw.addMutation(m); bw.close(); return 0; }
Mutation m = new Mutation(cloneTablet.getExtent().getMetadataEntry()); m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), k.getTimestamp()); bw.addMutation(m); bw.addMutation(createCloneMutation(srcTableId, tableId, st.getKeyValues())); Mutation m = new Mutation(cloneTablet.getExtent().getMetadataEntry()); m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK".getBytes(UTF_8))); bw.addMutation(m); bw.flush(); return rewrites;
Text start = extent.getPrevEndRow(); if (start == null) { start = new Text(); timeType = entry.getValue().toString().charAt(0); } else if (key.compareColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME) == 0) { throw new IllegalStateException( String path = entry.getValue().toString(); if (path.contains(":")) { datafiles.add(new FileRef(path)); BatchWriter bw = client.createBatchWriter(targetSystemTable, new BatchWriterConfig()); try { deleteTablets(info, deleteRange, bw, client); } finally { bw.close(); bw = client.createBatchWriter(targetSystemTable, new BatchWriterConfig()); try { Mutation m = new Mutation(followingTablet.getMetadataEntry()); TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(extent.getPrevEndRow())); ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m); bw.addMutation(m); bw.flush(); } finally { bw.close();