/** * Disable exploration on the given stream by dropping the Hive table for the stream. * * @param tableName name of the table to delete * @param streamId the ID of the stream to disable * @return the query handle for disabling the stream * @throws ExploreException if there was an exception dropping the table * @throws SQLException if there was a problem with the drop table statement */ public QueryHandle disableStream(String tableName, StreamId streamId) throws ExploreException, SQLException { LOG.debug("Disabling explore for stream {} with table {}", streamId, tableName); String deleteStatement = generateDeleteTableStatement(null, tableName); return exploreService.execute(streamId.getParent(), deleteStatement); }
@Override public StreamMetricsCollector createMetricsCollector(final StreamId streamId) { MetricsContext streamMetricsContext = streamMetricsCollectors.getUnchecked(streamId.getParent()); final MetricsContext childCollector = streamMetricsContext.childContext(Constants.Metrics.Tag.STREAM, streamId.getEntityName()); return new StreamMetricsCollector() { @Override public void emitMetrics(long bytesWritten, long eventsWritten) { if (bytesWritten > 0) { childCollector.increment("collect.bytes", bytesWritten); sizeCollector.received(streamId, bytesWritten); } if (eventsWritten > 0) { childCollector.increment("collect.events", eventsWritten); } } }; } };
/** * Returns the location for the given stream. */ private Location getStreamLocation(StreamId streamId) throws IOException { return getStreamBaseLocation(streamId.getParent()).append(streamId.getEntityName()); }
@Override public void updateConfig(final StreamId streamId, final StreamProperties properties) throws Exception { Location streamLocation; streamLocation = impersonator.doAs(streamId.getParent(), new Callable<Location>() { @Override public Location call() throws Exception {
return exploreService.execute(streamId.getParent(), createStatement);
streamId.getParent(), "SELECT * FROM " + getHiveTableName(streamId) + " WHERE " + timestampCol + " BETWEEN " + startTime + " AND " + endTime
@Override public synchronized StreamConsumerStateStore create(StreamConfig streamConfig) throws IOException { NamespaceId namespace = streamConfig.getStreamId().getParent(); TableId tableId = StreamUtils.getStateStoreTableId(namespace); getLevelDBTableAdmin(tableId).create(); String levelDBTableName = PrefixedNamespaces.namespace(cConf, tableId.getNamespace(), tableId.getTableName()); LevelDBTableCore coreTable = new LevelDBTableCore(levelDBTableName, tableService); return new LevelDBStreamConsumerStateStore(streamConfig, coreTable); }
@Override public synchronized StreamConsumerStateStore create(StreamConfig streamConfig) throws IOException { NamespaceId namespace = streamConfig.getStreamId().getParent(); TableId tableId = StreamUtils.getStateStoreTableId(namespace); InMemoryTableAdmin admin = new InMemoryTableAdmin(DatasetContext.from(tableId.getNamespace()), tableId.getTableName(), cConf); if (!admin.exists()) { admin.create(); } InMemoryTable table = new NoTxInMemoryTable(DatasetContext.from(tableId.getNamespace()), tableId.getTableName(), cConf); return new InMemoryStreamConsumerStateStore(streamConfig, table); }
@Override public synchronized StreamConsumerStateStore create(StreamConfig streamConfig) throws IOException { NamespaceId namespace = streamConfig.getStreamId().getParent(); TableId streamStateStoreTableId = StreamUtils.getStateStoreTableId(namespace); TableId hbaseTableId = tableUtil.createHTableId(new NamespaceId(streamStateStoreTableId.getNamespace()), streamStateStoreTableId.getTableName()); boolean tableExist; try (HBaseAdmin admin = new HBaseAdmin(hConf)) { tableExist = tableUtil.tableExists(admin, hbaseTableId); } if (!tableExist) { try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) { TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(hbaseTableId, cConf); ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf); tdBuilder.addColumnFamily(cfdBuilder.build()); ddlExecutor.createTableIfNotExists(tdBuilder.build(), null); } } HTable hTable = tableUtil.createHTable(hConf, hbaseTableId); hTable.setWriteBufferSize(Constants.Stream.HBASE_WRITE_BUFFER_SIZE); hTable.setAutoFlushTo(false); return new HBaseStreamConsumerStateStore(streamConfig, hTable); }