if (task.getTable().isPresent()) { String actualTableName = normalizeTableNameCase(con, task.getTable().get()); task.setTable(Optional.of(actualTableName)); if (task.getUseRawQueryWithIncremental()) { String temporaryQuery = rawQuery; for (int i = 0; i < task.getIncrementalColumns().size(); i++) { columnNames.put(task.getIncrementalColumns().get(i), i); task.setQuerySchema(querySchema); if (task.getIncremental()) { List<String> incrementalColumns = task.getIncrementalColumns(); if (incrementalColumns.isEmpty()) { if (!task.getTable().isPresent()) { throw new ConfigException("incremental_columns option must be set if incremental is true and custom query option is set"); List<String> primaryKeys = con.getPrimaryKeys(task.getTable().get()); if (primaryKeys.isEmpty()) { throw new ConfigException(String.format(ENGLISH, "Primary key is not available at the table '%s'. incremental_columns option must be set", task.getTable().get())); task.setIncrementalColumns(primaryKeys); incrementalColumns = primaryKeys;
PreparedQuery builtQuery = task.getBuiltQuery(); JdbcSchema querySchema = task.getQuerySchema(); BufferAllocator allocator = task.getBufferAllocator(); PageBuilder pageBuilder = new PageBuilder(allocator, schema, output); try (BatchSelect cursor = con.newSelectCursor(builtQuery, getters, task.getFetchRows(), task.getSocketTimeout())) { while (true) { long rows = fetch(cursor, getters, pageBuilder); if (task.getIncremental() && totalRows > 0) { lastRecordStore = new LastRecordStore(task.getIncrementalColumnIndexes(), task.getIncrementalColumns()); lastRecordStore.accept(getters); if (task.getAfterSelect().isPresent()) { con.executeUpdate(task.getAfterSelect().get()); con.connection.commit();
case "DATETIME": case "TIMESTAMP": int index = task.getQuerySchema().findColumn(column.getName()).get(); if (!task.getIncremental() || !task.getIncrementalColumnIndexes().contains(index)) { return getter;
@Override public ConfigDiff transaction(ConfigSource config, InputPlugin.Control control) { PluginTask task = config.loadConfig(getTaskClass()); if (task.getIncremental()) { if (task.getOrderBy().isPresent()) { throw new ConfigException("order_by option must not be set if incremental is true"); } } else { if (!task.getIncrementalColumns().isEmpty()) { throw new ConfigException("'incremental: true' must be set if incremental_columns is set"); } } Schema schema; try (JdbcInputConnection con = newConnection(task)) { con.showDriverVersion(); // TODO incremental_columns is not set => get primary key schema = setupTask(con, task); } catch (SQLException ex) { throw Throwables.propagate(ex); } return buildNextConfigDiff(task, control.run(task.dump(), schema, 1)); }
private List<ColumnGetter> newColumnGetters(JdbcInputConnection con, PluginTask task, JdbcSchema querySchema, PageBuilder pageBuilder) throws SQLException { ColumnGetterFactory factory = newColumnGetterFactory(pageBuilder, task.getDefaultTimeZone()); ImmutableList.Builder<ColumnGetter> getters = ImmutableList.builder(); for (JdbcColumn c : querySchema.getColumns()) { JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), c, factory.getJdbcType(c.getSqlType())); getters.add(factory.newColumnGetter(con, task, c, columnOption)); } return getters.build(); }