protected KuduSession getKuduSession(final KuduClient client) { final KuduSession kuduSession = client.newSession(); kuduSession.setMutationBufferSpace(batchSize); kuduSession.setFlushMode(flushMode); if (operationType == OperationType.INSERT_IGNORE) { kuduSession.setIgnoreAllDuplicateRows(true); } return kuduSession; }
@Override public void init() throws DBException { this.tableName = getProperties().getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT); initClient(); this.session = client.newSession(); if (getProperties().getProperty(SYNC_OPS_OPT) != null && getProperties().getProperty(SYNC_OPS_OPT).equals("false")) { this.session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_BACKGROUND); this.session.setMutationBufferSpace( getIntFromProp(getProperties(), BUFFER_NUM_OPS_OPT, BUFFER_NUM_OPS_DEFAULT)); } else { this.session.setFlushMode(KuduSession.FlushMode.AUTO_FLUSH_SYNC); } try { this.kuduTable = client.openTable(tableName); this.schema = kuduTable.getSchema(); } catch (Exception e) { throw new DBException("Could not open a table because of:", e); } }
@Override protected void setup(Context context) throws KuduException { id = "Job: " + context.getJobID() + " Task: " + context.getTaskAttemptID(); Configuration conf = context.getConfiguration(); CommandLineParser parser = new CommandLineParser(conf); client = parser.getClient(); table = client.openTable(getTableName(conf)); headsTable = client.openTable(getHeadsTable(conf)); session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); session.setMutationBufferSpace(WIDTH_DEFAULT); session.setIgnoreAllDuplicateRows(true); this.width = context.getConfiguration().getInt(GENERATOR_WIDTH_KEY, WIDTH_DEFAULT); current = new byte[this.width][]; int wrapMultiplier = context.getConfiguration().getInt(GENERATOR_WRAP_KEY, WRAP_DEFAULT); this.wrap = (long)wrapMultiplier * width; this.numNodes = context.getConfiguration().getLong( GENERATOR_NUM_ROWS_PER_MAP_KEY, (long)WIDTH_DEFAULT * WRAP_DEFAULT); if (this.numNodes < this.wrap) { this.wrap = this.numNodes; } }
KuduConnection(Config config, byte[] token) { client = new KuduClient.KuduClientBuilder(config.getString(CONNECTION_CONFIG_NAME)).build(); if (token != null) { client.importAuthenticationCredentials(token); } session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); session.setMutationBufferSpace(10000); session.setIgnoreAllDuplicateRows(KuduUtils.doesInsertIgnoreDuplicates(config)); }