protected KuduSession getKuduSession(final KuduClient client) { final KuduSession kuduSession = client.newSession(); kuduSession.setMutationBufferSpace(batchSize); kuduSession.setFlushMode(flushMode); if (operationType == OperationType.INSERT_IGNORE) { kuduSession.setIgnoreAllDuplicateRows(true); } return kuduSession; }
@Override protected void setup(Context context) throws KuduException { id = "Job: " + context.getJobID() + " Task: " + context.getTaskAttemptID(); Configuration conf = context.getConfiguration(); CommandLineParser parser = new CommandLineParser(conf); client = parser.getClient(); table = client.openTable(getTableName(conf)); headsTable = client.openTable(getHeadsTable(conf)); session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH); session.setMutationBufferSpace(WIDTH_DEFAULT); session.setIgnoreAllDuplicateRows(true); this.width = context.getConfiguration().getInt(GENERATOR_WIDTH_KEY, WIDTH_DEFAULT); current = new byte[this.width][]; int wrapMultiplier = context.getConfiguration().getInt(GENERATOR_WRAP_KEY, WRAP_DEFAULT); this.wrap = (long)wrapMultiplier * width; this.numNodes = context.getConfiguration().getLong( GENERATOR_NUM_ROWS_PER_MAP_KEY, (long)WIDTH_DEFAULT * WRAP_DEFAULT); if (this.numNodes < this.wrap) { this.wrap = this.numNodes; } }
KuduConnection(Config config, byte[] token) { client = new KuduClient.KuduClientBuilder(config.getString(CONNECTION_CONFIG_NAME)).build(); if (token != null) { client.importAuthenticationCredentials(token); } session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); session.setMutationBufferSpace(10000); session.setIgnoreAllDuplicateRows(KuduUtils.doesInsertIgnoreDuplicates(config)); }
@Test(timeout = 100000) public void testIgnoreAllDuplicateRows() throws Exception { KuduTable table = client.createTable(tableName, basicSchema, getBasicCreateTableOptions()); KuduSession session = client.newSession(); session.setIgnoreAllDuplicateRows(true); for (int i = 0; i < 10; i++) { session.apply(createInsert(table, i)); } for (SessionConfiguration.FlushMode mode : SessionConfiguration.FlushMode.values()) { session.setFlushMode(mode); for (int i = 0; i < 10; i++) { OperationResponse resp = session.apply(createInsert(table, i)); if (mode == SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC) { assertFalse(resp.hasRowError()); } } if (mode == SessionConfiguration.FlushMode.MANUAL_FLUSH) { List<OperationResponse> responses = session.flush(); for (OperationResponse resp : responses) { assertFalse(resp.hasRowError()); } } else if (mode == SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND) { while (session.hasPendingOperations()) { Thread.sleep(100); } assertEquals(0, session.countPendingErrors()); } } }