public StreamResultFuture stream() { return stream(Collections.<InetAddress>emptySet()); }
public void onSuccess(StreamState finalState) { releaseReferences(); } public void onFailure(Throwable t)
LoaderOptions options = LoaderOptions.parseArgs(args); OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug); SSTableLoader loader = new SSTableLoader( options.directory, new ExternalClient( future = loader.stream(options.ignores); future = loader.stream(options.ignores, indicator); System.err.println(loader.getFailedHosts()); e.printStackTrace(System.err); System.exit(1);
Future<StreamState> future = loader.stream(ignores); while (true) if (loader.getFailedHosts().size() > 0) if (loader.getFailedHosts().size() > maxFailures) throw new IOException("Too many hosts failed: " + loader.getFailedHosts()); else logger.warn("Some hosts failed: {}", loader.getFailedHosts());
return new SSTableLoader(dir, client, new OutputHandler.LogOutput()).stream();
public StreamResultFuture stream(Set<InetAddress> toIgnore, StreamEventHandler... listeners) { client.init(keyspace); outputHandler.output("Established connection to initial hosts"); StreamPlan plan = new StreamPlan("Bulk Load", 0, connectionsPerHost, false, false, false).connectionFactory(client.getConnectionFactory()); Map<InetAddress, Collection<Range<Token>>> endpointToRanges = client.getEndpointToRangesMap(); openSSTables(endpointToRanges); if (sstables.isEmpty()) { // return empty result return plan.execute(); } outputHandler.output(String.format("Streaming relevant part of %s to %s", names(sstables), endpointToRanges.keySet())); for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : endpointToRanges.entrySet()) { InetAddress remote = entry.getKey(); if (toIgnore.contains(remote)) continue; List<StreamSession.SSTableStreamingSections> endpointDetails = new LinkedList<>(); // references are acquired when constructing the SSTableStreamingSections above for (StreamSession.SSTableStreamingSections details : streamingDetails.get(remote)) { endpointDetails.add(details); } plan.transferFiles(remote, endpointDetails); } plan.listeners(this, listeners); return plan.execute(); }
private void prepareWriter() throws IOException { if (outputDir == null) { String keyspace = ConfigHelper.getOutputKeyspace(conf); //dir must be named by ks/cf for the loader outputDir = new File(getOutputLocation() + File.separator + keyspace + File.separator + ConfigHelper.getOutputColumnFamily(conf)); outputDir.mkdirs(); } if (writer == null) { AbstractType<?> subcomparator = null; if (cfType == CFType.SUPER) subcomparator = BytesType.instance; writer = new SSTableSimpleUnsortedWriter( outputDir, ConfigHelper.getOutputPartitioner(conf), ConfigHelper.getOutputKeyspace(conf), ConfigHelper.getOutputColumnFamily(conf), BytesType.instance, subcomparator, Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64")), ConfigHelper.getOutputCompressionParamaters(conf)); this.loader = new SSTableLoader(outputDir, new ExternalClient(conf), new NullOutputHandler()); } }
SSTableLoader loader = new SSTableLoader( options.directory.getAbsoluteFile(), new ExternalClient( future = loader.stream(options.ignores); future = loader.stream(options.ignores, indicator); System.err.println(loader.getFailedHosts()); e.printStackTrace(System.err); throw new BulkLoadException(e);
Future<StreamState> future = loader.stream(ignores); while (true) if (loader.getFailedHosts().size() > 0) if (loader.getFailedHosts().size() > maxFailures) throw new IOException("Too many hosts failed: " + loader.getFailedHosts()); else logger.warn("Some hosts failed: {}", loader.getFailedHosts());
return new SSTableLoader(dir, client, new OutputHandler.LogOutput()).stream();
public StreamResultFuture stream(Set<InetAddress> toIgnore, StreamEventHandler... listeners) { client.init(keyspace); outputHandler.output("Established connection to initial hosts"); StreamPlan plan = new StreamPlan("Bulk Load", 0, connectionsPerHost, false, false, false).connectionFactory(client.getConnectionFactory()); Map<InetAddress, Collection<Range<Token>>> endpointToRanges = client.getEndpointToRangesMap(); openSSTables(endpointToRanges); if (sstables.isEmpty()) { // return empty result return plan.execute(); } outputHandler.output(String.format("Streaming relevant part of %s to %s", names(sstables), endpointToRanges.keySet())); for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : endpointToRanges.entrySet()) { InetAddress remote = entry.getKey(); if (toIgnore.contains(remote)) continue; List<StreamSession.SSTableStreamingSections> endpointDetails = new LinkedList<>(); // references are acquired when constructing the SSTableStreamingSections above for (StreamSession.SSTableStreamingSections details : streamingDetails.get(remote)) { endpointDetails.add(details); } plan.transferFiles(remote, endpointDetails); } plan.listeners(this, listeners); return plan.execute(); }
private void prepareWriter() throws IOException { if (outputdir == null) { String keyspace = ConfigHelper.getOutputKeyspace(conf); //dir must be named by ks/cf for the loader outputdir = new File(getOutputLocation() + File.separator + keyspace + File.separator + ConfigHelper.getOutputColumnFamily(conf)); outputdir.mkdirs(); } if (writer == null) { AbstractType<?> subcomparator = null; ExternalClient externalClient = null; String username = ConfigHelper.getOutputKeyspaceUserName(conf); String password = ConfigHelper.getOutputKeyspacePassword(conf); if (cfType == CFType.SUPER) subcomparator = BytesType.instance; this.writer = new SSTableSimpleUnsortedWriter( outputdir, ConfigHelper.getOutputPartitioner(conf), ConfigHelper.getOutputKeyspace(conf), ConfigHelper.getOutputColumnFamily(conf), BytesType.instance, subcomparator, Integer.parseInt(conf.get(BUFFER_SIZE_IN_MB, "64")), ConfigHelper.getOutputCompressionParamaters(conf)); externalClient = new ExternalClient(ConfigHelper.getOutputInitialAddress(conf), ConfigHelper.getOutputRpcPort(conf), username, password); this.loader = new SSTableLoader(outputdir, externalClient, new NullOutputHandler()); } }
SSTableLoader loader = new SSTableLoader( options.directory.getAbsoluteFile(), new ExternalClient( future = loader.stream(options.ignores); future = loader.stream(options.ignores, indicator); System.err.println(loader.getFailedHosts()); e.printStackTrace(System.err); throw new BulkLoadException(e);
private void close() throws IOException { if (writer != null) { writer.close(); Future<StreamState> future = loader.stream(); while (true) { try { future.get(1000, TimeUnit.MILLISECONDS); break; } catch (ExecutionException | TimeoutException te) { if (null != progress) progress.progress(); if (null != context) HadoopCompat.progress(context); } catch (InterruptedException e) { throw new IOException(e); } } if (loader.getFailedHosts().size() > 0) { if (loader.getFailedHosts().size() > maxFailures) throw new IOException("Too many hosts failed: " + loader.getFailedHosts()); else logger.warn("Some hosts failed: " + loader.getFailedHosts()); } } }
SSTableLoader loader = new SSTableLoader(dir, client, new OutputHandler.LogOutput()); return loader.stream();
public StreamResultFuture stream(Set<InetAddress> toIgnore, StreamEventHandler... listeners) { client.init(keyspace); outputHandler.output("Established connection to initial hosts"); StreamPlan plan = new StreamPlan("Bulk Load", 0, connectionsPerHost, false, false, false).connectionFactory(client.getConnectionFactory()); Map<InetAddress, Collection<Range<Token>>> endpointToRanges = client.getEndpointToRangesMap(); openSSTables(endpointToRanges); if (sstables.isEmpty()) { // return empty result return plan.execute(); } outputHandler.output(String.format("Streaming relevant part of %s to %s", names(sstables), endpointToRanges.keySet())); for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : endpointToRanges.entrySet()) { InetAddress remote = entry.getKey(); if (toIgnore.contains(remote)) continue; List<StreamSession.SSTableStreamingSections> endpointDetails = new LinkedList<>(); // references are acquired when constructing the SSTableStreamingSections above for (StreamSession.SSTableStreamingSections details : streamingDetails.get(remote)) { endpointDetails.add(details); } plan.transferFiles(remote, endpointDetails); } plan.listeners(this, listeners); return plan.execute(); }
public void onSuccess(StreamState finalState) { releaseReferences(); } public void onFailure(Throwable t)
public StreamResultFuture stream() { return stream(Collections.<InetAddress>emptySet()); }
SSTableLoader loader = new SSTableLoader( options.directory.getAbsoluteFile(), new ExternalClient( future = loader.stream(options.ignores); future = loader.stream(options.ignores, indicator); System.err.println(loader.getFailedHosts()); e.printStackTrace(System.err); throw new BulkLoadException(e);
Future<StreamState> future = loader.stream(ignores); while (true) if (loader.getFailedHosts().size() > 0) if (loader.getFailedHosts().size() > maxFailures) throw new IOException("Too many hosts failed: " + loader.getFailedHosts()); else logger.warn("Some hosts failed: {}", loader.getFailedHosts());