/** * Use this method for creating a PartitionMap that has only one shard (0) and whose range is -Infinity, +Infinity */ public static PartitionMap oneShardOpenedMap() { // For convenience, we create a 1-shard opened map here (shard = 0) List<PartitionEntry> partitionMap = new ArrayList<PartitionEntry>(); PartitionEntry openedEntry = new PartitionEntry(); openedEntry.setMin(null); openedEntry.setMax(null); openedEntry.setShard(0); partitionMap.add(openedEntry); return new PartitionMap(partitionMap); }
/** * Given a min and a max key, return all the partitions that impact this range, min and max inclusive. * <p> * Note that (null, null) is a valid input to this method and will be interpreted as the whole key range, regardless * of the key type (that's why we use null for representing opened ranges). */ public List<Integer> findPartitions(String minKey, String maxKey) { List<Integer> partitions = new ArrayList<Integer>(); int minPartitionIndex = 0, maxPartitionIndex = partitionEntries.size() - 1; if(minKey != null) { minPartitionIndex = findPartition(minKey); } if(maxKey != null) { maxPartitionIndex = findPartition(maxKey); } for(int i = minPartitionIndex; i <= maxPartitionIndex; i++) { partitions.add(partitionEntries.get(i).getShard()); } return partitions; }
public void start() throws InterruptedException, IOException { Map<String, Object> context = new HashMap<String, Object>(); context.put("qnodes", qNodes); Map<Integer, Long> rowIdsPerPartition = new HashMap<Integer, Long>(); context.put("rowIdsPerPartition", rowIdsPerPartition); // First get Tablespace metadata - nPartitions SploutClient client = new SploutClient(((String) context.get("qnodes")).split(",")); Tablespace tablespace = client.tablespace(TABLESPACE); int nPartitions = tablespace.getPartitionMap().getPartitionEntries().size(); // Then gather number of registers for each partition for (int partition = 0; partition < nPartitions; partition++) { String query = "SELECT MAX(rowid) FROM pagecounts;"; Map<String, Object> obj = (Map<String, Object>) client .query(TABLESPACE, null, query, partition + "").getResult().get(0); rowIdsPerPartition.put(partition, Long.parseLong(obj.get("MAX(rowid)").toString())); } SploutBenchmark benchmark = new SploutBenchmark(); for (int i = 0; i < nIterations; i++) { benchmark.stressTest(nThreads, nQueries, PageCountsStressThreadImpl.class, context); benchmark.printStats(System.out); } }
partitionId = partitionMap.findPartition(key); if (partitionId == PartitionMap.NO_PARTITION) { return new ErrorQueryStatus("Key out of partition ranges: " + key + " for tablespace " + tablespaceName); partitionId = (int) (Math.random() * partitionMap.getPartitionEntries().size()); } else { try {
tablespace.getPartitionMap().getPartitionEntries().remove(pEntry); partitionMap.addAll(currentTablespace.getPartitionMap().getPartitionEntries()); replicationMap.addAll(currentTablespace.getReplicationMap().getReplicationEntries()); deployDate = currentTablespace.getCreationDate(); } else { currentTablespace = new Tablespace(new PartitionMap(partitionMap), new ReplicationMap(replicationMap), versionName, deployDate); tablespaceVersionMap.put(tablespaceVersion, currentTablespace);
@Override public void generateView(Configuration conf, SamplingType samplingType, SamplingOptions samplingOptions) throws Exception { prepareOutput(conf); final int nPartitions = tablespace.getnPartitions(); if (nPartitions > 1) { partitionMap = sample(nPartitions, conf, samplingType, samplingOptions); } else { partitionMap = PartitionMap.oneShardOpenedMap(); } writeOutputMetadata(conf); TupleMRBuilder builder = createMRBuilder(nPartitions, conf); // Set a TupleOutput here instead of SQLiteOutput builder.setOutput(new Path(outputPath, OUT_STORE), new TupleOutputFormat(tableSchema), ITuple.class, NullWritable.class); executeViewGeneration(builder); } };
impactedKeys.addAll(tablespace.getPartitionMap().findPartitions(null, null)); // all partitions are hit impactedKeys.addAll(tablespace.getPartitionMap().findPartitions(keyMins.get(i), keyMaxs.get(i)));
/** * Launches the generation of the tablespaces. Automatic * {@link com.splout.db.common.PartitionMap} calculation. * * This is the public method which has to be called when using this class as * an API. Business logic has been split in various protected functions to * ease understading of it and also to be able to subclass this easily to * extend its functionality. */ public void generateView(Configuration conf, TupleSampler.SamplingType samplingType, TupleSampler.SamplingOptions samplingOptions) throws Exception { prepareOutput(conf); final int nPartitions = tablespace.getnPartitions(); if (nPartitions > 1) { partitionMap = sample(nPartitions, conf, samplingType, samplingOptions); } else { partitionMap = PartitionMap.oneShardOpenedMap(); } Log.info("Calculated partition map: " + partitionMap); writeOutputMetadata(conf); TupleMRBuilder builder = createMRBuilder(nPartitions, conf); executeViewGeneration(builder); }
public DeployInfo deploy(String tablespace, PartitionMap partitionMap, ReplicationMap replicationMap, URI dataUri) throws IOException { DeployRequest deployRequest = new DeployRequest(); deployRequest.setTablespace(tablespace); deployRequest.setData_uri(dataUri.toString()); deployRequest.setPartitionMap(partitionMap.getPartitionEntries()); deployRequest.setReplicationMap(replicationMap.getReplicationEntries()); return deploy(deployRequest); }
/** * Use this method for creating a PartitionMap that has only one shard (0) and whose range is -Infinity, +Infinity */ public static PartitionMap oneShardOpenedMap() { // For convenience, we create a 1-shard opened map here (shard = 0) List<PartitionEntry> partitionMap = new ArrayList<PartitionEntry>(); PartitionEntry openedEntry = new PartitionEntry(); openedEntry.setMin(null); openedEntry.setMax(null); openedEntry.setShard(0); partitionMap.add(openedEntry); return new PartitionMap(partitionMap); }
/** * Given a min and a max key, return all the partitions that impact this range, min and max inclusive. * <p/> * Note that (null, null) is a valid input to this method and will be interpreted as the whole key range, regardless * of the key type (that's why we use null for representing opened ranges). */ public List<Integer> findPartitions(String minKey, String maxKey) { List<Integer> partitions = new ArrayList<Integer>(); int minPartitionIndex = 0, maxPartitionIndex = partitionEntries.size() - 1; if (minKey != null) { minPartitionIndex = findPartition(minKey); } if (maxKey != null) { maxPartitionIndex = findPartition(maxKey); } for (int i = minPartitionIndex; i <= maxPartitionIndex; i++) { partitions.add(partitionEntries.get(i).getShard()); } return partitions; }
/** * Launches the generation of the tablespaces. Uses the custom partition map * provided. * * This is the public method which has to be called when using this class as * an API. Business logic has been split in various protected functions to * ease understading of it and also to be able to subclass this easily to * extend its functionality. */ public void generateView(Configuration conf, PartitionMap partitionMap) throws Exception { prepareOutput(conf); this.partitionMap = partitionMap; final int nPartitions = partitionMap.getPartitionEntries().size(); Log.info("Using provided partition map: " + partitionMap); writeOutputMetadata(conf); TupleMRBuilder builder = createMRBuilder(nPartitions, conf); executeViewGeneration(builder); }
return new PartitionMap(partitionEntries);
shardId = partitionMap.findPartition(strKey); if (shardId == PartitionMap.NO_PARTITION) { throw new RuntimeException(
deployRequests[tIndex].setPartitionMap(partitionMap.getPartitionEntries()); deployRequests[tIndex].setReplicationMap(ReplicationMap.roundRobinMap(partitionMap.getPartitionEntries().size(), repFactor, dnodes.toArray(new String[0])).getReplicationEntries());
partitionEntries.get(partitionEntries.size() - 1).setMax(null); PartitionMap partitionMap = new PartitionMap(partitionEntries); HadoopUtils.stringToFile(outFs, new Path(out, "partition-map"), JSONSerDe.ser(partitionMap));
public DeployInfo deploy(String tablespace, PartitionMap partitionMap, ReplicationMap replicationMap, URI dataUri) throws IOException { DeployRequest deployRequest = new DeployRequest(); deployRequest.setTablespace(tablespace); deployRequest.setData_uri(dataUri.toString()); deployRequest.setPartitionMap(partitionMap.getPartitionEntries()); deployRequest.setReplicationMap(replicationMap.getReplicationEntries()); return deploy(deployRequest); }
int nPartitions = tablespace.getPartitionMap().getPartitionEntries().size();
&& t.getPartitionMap().getPartitionEntries() != null && t.getReplicationMap().getReplicationEntries() != null && t.getReplicationMap().getReplicationEntries().size() > 0) { if (t.getPartitionMap().getPartitionEntries().size() == t.getReplicationMap() .getReplicationEntries().size()) { log.info("Ok, TablespaceVersion [" + req.getTablespace() + ", " + req.getVersion()