private Operator createTableWriterOperator(BlockingPageSink blockingPageSink) { PageSinkManager pageSinkManager = new PageSinkManager(); pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(blockingPageSink)); return createTableWriterOperator(pageSinkManager, new DevNullOperatorFactory(1, new PlanNodeId("test")), ImmutableList.of(BIGINT, VARBINARY)); }
private ConnectorPageSink createPageSink() { if (target instanceof CreateHandle) { return pageSinkManager.createPageSink(session, ((CreateHandle) target).getHandle()); } if (target instanceof InsertHandle) { return pageSinkManager.createPageSink(session, ((InsertHandle) target).getHandle()); } throw new UnsupportedOperationException("Unhandled target type: " + target.getClass().getName()); }
@Override public ConnectorPageSink createPageSink(Session session, OutputTableHandle tableHandle) { // assumes connectorId and catalog are the same ConnectorSession connectorSession = session.toConnectorSession(tableHandle.getConnectorId()); return providerFor(tableHandle.getConnectorId()).createPageSink(tableHandle.getTransactionHandle(), connectorSession, tableHandle.getConnectorHandle()); }
nodeSchedulerConfig, new NodeTaskMap(finalizerService)); this.pageSinkManager = new PageSinkManager(); CatalogManager catalogManager = new CatalogManager(); this.transactionManager = InMemoryTransactionManager.create(
private synchronized void addConnectorInternal(MaterializedConnector connector) { checkState(!stopped.get(), "ConnectorManager is stopped"); ConnectorId connectorId = connector.getConnectorId(); checkState(!connectors.containsKey(connectorId), "A connector %s already exists", connectorId); connectors.put(connectorId, connector); splitManager.addConnectorSplitManager(connectorId, connector.getSplitManager()); pageSourceManager.addConnectorPageSourceProvider(connectorId, connector.getPageSourceProvider()); connector.getPageSinkProvider() .ifPresent(pageSinkProvider -> pageSinkManager.addConnectorPageSinkProvider(connectorId, pageSinkProvider)); connector.getIndexProvider() .ifPresent(indexProvider -> indexManager.addIndexProvider(connectorId, indexProvider)); connector.getPartitioningProvider() .ifPresent(partitioningProvider -> nodePartitioningManager.addPartitioningProvider(connectorId, partitioningProvider)); metadataManager.getProcedureRegistry().addProcedures(connectorId, connector.getProcedures()); connector.getAccessControl() .ifPresent(accessControl -> accessControlManager.addCatalogAccessControl(connectorId, accessControl)); metadataManager.getTablePropertyManager().addProperties(connectorId, connector.getTableProperties()); metadataManager.getColumnPropertyManager().addProperties(connectorId, connector.getColumnProperties()); metadataManager.getSchemaPropertyManager().addProperties(connectorId, connector.getSchemaProperties()); metadataManager.getSessionPropertyManager().addConnectorSessionProperties(connectorId, connector.getSessionProperties()); }
private synchronized void removeConnectorInternal(ConnectorId connectorId) { splitManager.removeConnectorSplitManager(connectorId); pageSourceManager.removeConnectorPageSourceProvider(connectorId); pageSinkManager.removeConnectorPageSinkProvider(connectorId); indexManager.removeIndexProvider(connectorId); nodePartitioningManager.removePartitioningProvider(connectorId); metadataManager.getProcedureRegistry().removeProcedures(connectorId); accessControlManager.removeCatalogAccessControl(connectorId); metadataManager.getTablePropertyManager().removeProperties(connectorId); metadataManager.getColumnPropertyManager().removeProperties(connectorId); metadataManager.getSchemaPropertyManager().removeProperties(connectorId); metadataManager.getSessionPropertyManager().removeConnectorSessionProperties(connectorId); MaterializedConnector materializedConnector = connectors.remove(connectorId); if (materializedConnector != null) { Connector connector = materializedConnector.getConnector(); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(connector.getClass().getClassLoader())) { connector.shutdown(); } catch (Throwable t) { log.error(t, "Error shutting down connector: %s", connectorId); } } }
new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata, pageFunctionCompiler),
pageSinkManager.addConnectorPageSinkProvider(connectorId, connectorPageSinkProvider);
@Test public void testTableWriterInfo() { PageSinkManager pageSinkManager = new PageSinkManager(); pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(new TableWriteInfoTestPageSink())); TableWriterOperator tableWriterOperator = (TableWriterOperator) createTableWriterOperator( pageSinkManager, new DevNullOperatorFactory(1, new PlanNodeId("test")), ImmutableList.of(BIGINT, VARBINARY)); RowPagesBuilder rowPagesBuilder = rowPagesBuilder(BIGINT); for (int i = 0; i < 100; i++) { rowPagesBuilder.addSequencePage(100, 0); } List<Page> pages = rowPagesBuilder.build(); long peakMemoryUsage = 0; long validationCpuNanos = 0; for (int i = 0; i < pages.size(); i++) { Page page = pages.get(i); peakMemoryUsage += page.getRetainedSizeInBytes(); validationCpuNanos += page.getPositionCount(); tableWriterOperator.addInput(page); TableWriterInfo info = tableWriterOperator.getInfo(); assertEquals(info.getPageSinkPeakMemoryUsage(), peakMemoryUsage); assertEquals((long) (info.getValidationCpuTime().getValue(NANOSECONDS)), validationCpuNanos); } }
new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)); this.pageSinkManager = new PageSinkManager(); this.transactionManager = TransactionManager.create( new TransactionManagerConfig().setIdleTimeout(new Duration(1, TimeUnit.DAYS)),
@Override public ConnectorPageSink createPageSink(Session session, InsertTableHandle tableHandle) { // assumes connectorId and catalog are the same ConnectorSession connectorSession = session.toConnectorSession(tableHandle.getConnectorId()); return providerFor(tableHandle.getConnectorId()).createPageSink(tableHandle.getTransactionHandle(), connectorSession, tableHandle.getConnectorHandle()); }
private ConnectorPageSink createPageSink() { if (target instanceof CreateHandle) { return pageSinkManager.createPageSink(session, ((CreateHandle) target).getHandle()); } if (target instanceof InsertHandle) { return pageSinkManager.createPageSink(session, ((InsertHandle) target).getHandle()); } throw new UnsupportedOperationException("Unhandled target type: " + target.getClass().getName()); }
throws Exception PageSinkManager pageSinkManager = new PageSinkManager(); pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(new TableWriteInfoTestPageSink())); ImmutableList<Type> outputTypes = ImmutableList.of(BIGINT, VARBINARY, BIGINT); Session session = testSessionBuilder()
public static LocalExecutionPlanner createTestingPlanner() { MetadataManager metadata = MetadataManager.createTestMetadataManager(); PageSourceManager pageSourceManager = new PageSourceManager(); pageSourceManager.addConnectorPageSourceProvider("test", new TestingPageSourceProvider()); // we don't start the finalizer so nothing will be collected, which is ok for a test FinalizerService finalizerService = new FinalizerService(); NodeScheduler nodeScheduler = new NodeScheduler( new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)); NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler); return new LocalExecutionPlanner( metadata, new SqlParser(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata), new IndexJoinLookupStats(), new CompilerConfig(), new TaskManagerConfig()); }
@Override public ConnectorPageSink createPageSink(Session session, OutputTableHandle tableHandle) { // assumes connectorId and catalog are the same ConnectorSession connectorSession = session.toConnectorSession(tableHandle.getConnectorId()); return providerFor(tableHandle.getConnectorId()).createPageSink(tableHandle.getTransactionHandle(), connectorSession, tableHandle.getConnectorHandle()); }
private Operator createTableWriterOperator(BlockingPageSink blockingPageSink) { PageSinkManager pageSinkProvider = new PageSinkManager(); pageSinkProvider.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(blockingPageSink)); TableWriterOperator.TableWriterOperatorFactory factory = new TableWriterOperator.TableWriterOperatorFactory( 0, new PlanNodeId("test"), pageSinkProvider, new TableWriterNode.CreateHandle(new OutputTableHandle( CONNECTOR_ID, new ConnectorTransactionHandle() {}, new ConnectorOutputTableHandle() {})), ImmutableList.of(0), Optional.empty(), TEST_SESSION); return factory.createOperator(createTaskContext(executor, TEST_SESSION) .addPipelineContext(true, true) .addDriverContext()); }
@Override public ConnectorPageSink createPageSink(Session session, InsertTableHandle tableHandle) { // assumes connectorId and catalog are the same ConnectorSession connectorSession = session.toConnectorSession(tableHandle.getConnectorId()); return providerFor(tableHandle.getConnectorId()).createPageSink(tableHandle.getTransactionHandle(), connectorSession, tableHandle.getConnectorHandle()); }