public XPartition withLocation(String value) { setLocation(value); return this; }
public XPartition withFactOrDimensionTableName(String value) { setFactOrDimensionTableName(value); return this; }
public XPartition withTimePartitionSpec(XTimePartSpec value) { setTimePartitionSpec(value); return this; }
public static StoragePartitionDesc storagePartSpecFromXPartition( XPartition xpart) { StoragePartitionDesc partDesc = new StoragePartitionDesc( xpart.getFactOrDimensionTableName(), timePartSpecfromXTimePartSpec(xpart.getTimePartitionSpec()), nonTimePartSpecfromXNonTimePartSpec(xpart.getNonTimePartitionSpec()), UpdatePeriod.valueOf(xpart.getUpdatePeriod().name())); partDesc.setPartParams(mapFromXProperties(xpart.getPartitionParameters())); partDesc.setSerdeParams(mapFromXProperties(xpart.getSerdeParameters())); partDesc.setLocation(xpart.getLocation()); partDesc.setInputFormat(xpart.getInputFormat()); partDesc.setOutputFormat(xpart.getOutputFormat()); partDesc.setSerializationLib(xpart.getSerdeClassname()); return partDesc; }
public static XPartition xpartitionFromPartition(String cubeTableName, Partition p, List<String> timePartCols) throws HiveException { XPartition xp = new XPartition(); xp.setFactOrDimensionTableName(cubeTableName); xp.setPartitionParameters(new XProperties()); xp.setSerdeParameters(new XProperties()); xp.setName(p.getCompleteName()); xp.setLocation(p.getLocation()); xp.setInputFormat(p.getInputFormatClass().getCanonicalName()); xp.setOutputFormat(p.getOutputFormatClass().getCanonicalName()); xp.getPartitionParameters().getProperty().addAll(xPropertiesFromMap(p.getParameters())); String upParam = p.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD); xp.setUpdatePeriod(XUpdatePeriod.valueOf(upParam)); LinkedHashMap<String, String> partSpec = p.getSpec(); xp.setFullPartitionSpec(new XPartSpec()); for (Map.Entry<String, String> entry : partSpec.entrySet()) { XPartSpecElement e = new XPartSpecElement(); e.setKey(entry.getKey()); e.setValue(entry.getValue()); xp.getFullPartitionSpec().getPartSpecElement().add(e); xp.setTimePartitionSpec(new XTimePartSpec()); xp.setNonTimePartitionSpec(new XPartSpec()); for (Map.Entry<String, String> entry : partSpec.entrySet()) { if (timePartCols.contains(entry.getKey())) { timePartSpecElement.setKey(entry.getKey()); timePartSpecElement .setValue(getXMLGregorianCalendar(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).parse( entry.getValue())));
private XPartition createPartition(String cubeTableName, final List<XTimePartSpecElement> timePartSpecs, String updatePeriod) { XPartition xp = cubeObjectFactory.createXPartition(); xp.setLocation(new Path(new File("target").getAbsolutePath(), "part/test_part").toString()); xp.setFactOrDimensionTableName(cubeTableName); xp.setNonTimePartitionSpec(new XPartSpec()); xp.setTimePartitionSpec(new XTimePartSpec()); xp.setPartitionParameters(new XProperties()); xp.setSerdeParameters(new XProperties()); for (XTimePartSpecElement timePartSpec : timePartSpecs) { xp.getTimePartitionSpec().getPartSpecElement().add(timePartSpec); } if (updatePeriod==null) { xp.setUpdatePeriod(XUpdatePeriod.HOURLY); } else { XUpdatePeriod updatePeriod1=XUpdatePeriod.valueOf(updatePeriod.toUpperCase()); xp.setUpdatePeriod(updatePeriod1); } return xp; }
assertEquals(partAddResult.getStatus(), Status.FAILED); xp.setLocation(xp.getLocation() + "/a/b/c"); APIResult partUpdateResult = target().path("metastore/facts/").path(table).path("storages/S1/partition") .queryParam("sessionid", lensSessionId).request(mediaType) assertEquals(partitions.getPartition().size(), 1); XPartition readPartition = partitions.getPartition().get(0); assertEquals(readPartition.getLocation(), xp.getLocation()); assertEquals(readPartition.getTimePartitionSpec(), xp.getTimePartitionSpec()); assertEquals(readPartition.getNonTimePartitionSpec(), xp.getNonTimePartitionSpec()); assertNotNull(readPartition.getFullPartitionSpec()); XTimePartSpecElement timePartSpec = readPartition.getTimePartitionSpec().getPartSpecElement().iterator().next(); XPartSpecElement fullPartSpec = readPartition.getFullPartitionSpec().getPartSpecElement().iterator().next(); assertEquals(timePartSpec.getKey(), fullPartSpec.getKey()); assertEquals( UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).format(JAXBUtils.getDateFromXML(timePartSpec.getValue())), fullPartSpec.getValue()); DateTime date = target().path("metastore/cubes").path("testCube").path("latestdate")
/** * Sets up query command instances and adds partitions to a table. * @throws Exception */ @BeforeClass public void setup() throws Exception { LensClient client = new LensClient(); LensCubeCommands command = new LensCubeCommands(); command.setClient(client); log.debug("Starting to test cube commands"); URL cubeSpec = TestLensQueryCommands.class.getClassLoader().getResource("schema/cubes/base/sample-cube.xml"); command.createCube(new File(cubeSpec.toURI())); TestLensDimensionCommands.createDimension(); TestLensDimensionTableCommands.addDim1Table("dim_table", "schema/dimtables/dim_table.xml", "local"); // Add partition URL dataDir = TestLensQueryCommands.class.getClassLoader().getResource("dim2-part"); XPartition xp = new XPartition(); xp.setFactOrDimensionTableName("dim_table"); xp.setLocation(new Path(dataDir.toURI()).toString()); xp.setUpdatePeriod(XUpdatePeriod.HOURLY); XTimePartSpec timePart = new XTimePartSpec(); XTimePartSpecElement partElement = new XTimePartSpecElement(); partElement.setKey("dt"); partElement.setValue(DatatypeFactory.newInstance().newXMLGregorianCalendar(new GregorianCalendar())); timePart.getPartSpecElement().add(partElement); xp.setTimePartitionSpec(timePart); APIResult result = client.addPartitionToDim("dim_table", "local", xp); assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED); }
public static void updatePartitionFromXPartition(Partition partition, XPartition xp) throws ClassNotFoundException { partition.getParameters().putAll(mapFromXProperties(xp.getPartitionParameters())); partition.getTPartition().getSd().getSerdeInfo().setParameters(mapFromXProperties(xp.getSerdeParameters())); partition.setLocation(xp.getLocation()); if (xp.getInputFormat() != null) { partition.setInputFormatClass(Class.forName(xp.getInputFormat()).asSubclass(InputFormat.class)); } if (xp.getOutputFormat() != null) { Class<? extends HiveOutputFormat> outputFormatClass = Class.forName(xp.getOutputFormat()).asSubclass(HiveOutputFormat.class); partition.setOutputFormatClass(outputFormatClass); } partition.getParameters().put(MetastoreConstants.PARTITION_UPDATE_PERIOD, xp.getUpdatePeriod().name()); partition.getTPartition().getSd().getSerdeInfo().setSerializationLib(xp.getSerdeClassname()); }
private void createFactPartition(String fileName, String fact, String storage) throws JAXBException, IOException { XPartition partition = (XPartition) SampleMetastore.readFromXML(fileName); String partLocation = partition.getLocation(); if (!partLocation.startsWith("/")) { partition.setLocation("file://" + System.getProperty("lens.home") + "/" + partLocation); } result = metaClient.addPartitionToFactTable(fact, storage, partition); if (result.getStatus().equals(APIResult.Status.FAILED)) { System.err.println("Adding partition from:" + fileName + " failed"); retCode = 1; } else { System.out.println("Added partition from:" + fileName); } }
public static Map<String, String> getFullPartSpecAsMap(XPartition partition) { Map<String, String> spec = Maps.newHashMap(); if (partition.getTimePartitionSpec() != null) { for (XTimePartSpecElement timePartSpecElement : partition.getTimePartitionSpec().getPartSpecElement()) { spec.put(timePartSpecElement.getKey(), UpdatePeriod.valueOf(partition.getUpdatePeriod().name()).format() .format(getDateFromXML(timePartSpecElement.getValue()))); } } if (partition.getNonTimePartitionSpec() != null) { for (XPartSpecElement partSpecElement : partition.getNonTimePartitionSpec().getPartSpecElement()) { spec.put(partSpecElement.getKey(), partSpecElement.getValue()); } } return spec; } }
/** * Add a new partition for a storage of dimension * * @param sessionid The sessionid in which user is working * @param dimTableName dimension table name * @param storage storage name * @param partition {@link XPartition} representation of partition * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if add was successful. {@link APIResult} with state * {@link Status#FAILED}, if add has failed */ @POST @Path("/dimtables/{dimTableName}/storages/{storage}/partition") public APIResult addPartitionToDimStorage(@QueryParam("sessionid") LensSessionHandle sessionid, @PathParam("dimTableName") String dimTableName, @PathParam("storage") String storage, XPartition partition) throws LensException { checkSessionId(sessionid); checkNonNullArgs("Partition is null", partition); checkNonNullArgs("Partition elements are null", partition.getFactOrDimensionTableName(), partition.getUpdatePeriod()); return successOrPartialOrFailure(getSvc().addPartitionToDimStorage(sessionid, dimTableName, storage, partition), 1); }
public XPartition withUpdatePeriod(XUpdatePeriod value) { setUpdatePeriod(value); return this; }
/** * Create an instance of {@link XPartition } * */ public XPartition createXPartition() { return new XPartition(); }
@Override public void updatePartitions(LensSessionHandle sessionid, String tblName, String storageName, XPartitionList xPartitions) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient client = getClient(sessionid); Set<String> storageTableNames = getAllTablesForStorage(sessionid, tblName, storageName); Map<UpdatePeriod, List<Partition>> partitionsToUpdate = new HashMap<>(); for (String storageTableName : storageTableNames) { for (XPartition xPartition : xPartitions.getPartition()) { Partition existingPartition = client.getPartitionByFilter(storageTableName, StorageConstants.getPartFilter(JAXBUtils.getFullPartSpecAsMap(xPartition))); JAXBUtils.updatePartitionFromXPartition(existingPartition, xPartition); UpdatePeriod updatePeriod = UpdatePeriod.valueOf(xPartition.getUpdatePeriod().value()); List<Partition> partitionList = partitionsToUpdate.get(updatePeriod); if (partitionList == null) { partitionList = new ArrayList<>(); partitionsToUpdate.put(updatePeriod, partitionList); } partitionList.add(existingPartition); } } client.updatePartitions(tblName, storageName, partitionsToUpdate); } catch (HiveException | ClassNotFoundException | InvalidOperationException exc) { throw new LensException(exc); } }
public static StoragePartitionDesc storagePartSpecFromXPartition( XPartition xpart) { StoragePartitionDesc partDesc = new StoragePartitionDesc( xpart.getFactOrDimensionTableName(), timePartSpecfromXTimePartSpec(xpart.getTimePartitionSpec()), nonTimePartSpecfromXNonTimePartSpec(xpart.getNonTimePartitionSpec()), UpdatePeriod.valueOf(xpart.getUpdatePeriod().name())); partDesc.setPartParams(mapFromXProperties(xpart.getPartitionParameters())); partDesc.setSerdeParams(mapFromXProperties(xpart.getSerdeParameters())); partDesc.setLocation(xpart.getLocation()); partDesc.setInputFormat(xpart.getInputFormat()); partDesc.setOutputFormat(xpart.getOutputFormat()); partDesc.setSerializationLib(xpart.getSerdeClassname()); return partDesc; }
public static XPartition xpartitionFromPartition(String cubeTableName, Partition p, List<String> timePartCols) throws HiveException { XPartition xp = new XPartition(); xp.setFactOrDimensionTableName(cubeTableName); xp.setPartitionParameters(new XProperties()); xp.setSerdeParameters(new XProperties()); xp.setName(p.getCompleteName()); xp.setLocation(p.getLocation()); xp.setInputFormat(p.getInputFormatClass().getCanonicalName()); xp.setOutputFormat(p.getOutputFormatClass().getCanonicalName()); xp.getPartitionParameters().getProperty().addAll(xPropertiesFromMap(p.getParameters())); String upParam = p.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD); xp.setUpdatePeriod(XUpdatePeriod.valueOf(upParam)); LinkedHashMap<String, String> partSpec = p.getSpec(); xp.setFullPartitionSpec(new XPartSpec()); for (Map.Entry<String, String> entry : partSpec.entrySet()) { XPartSpecElement e = new XPartSpecElement(); e.setKey(entry.getKey()); e.setValue(entry.getValue()); xp.getFullPartitionSpec().getPartSpecElement().add(e); xp.setTimePartitionSpec(new XTimePartSpec()); xp.setNonTimePartitionSpec(new XPartSpec()); for (Map.Entry<String, String> entry : partSpec.entrySet()) { if (timePartCols.contains(entry.getKey())) { timePartSpecElement.setKey(entry.getKey()); timePartSpecElement .setValue(getXMLGregorianCalendar(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).parse( entry.getValue())));
xp.setLocation(xp.getLocation() + "/a/b/c"); APIResult partUpdateResult = target().path("metastore/dimtables/").path(table).path("storages/test/partition") .queryParam("sessionid", lensSessionId).request(mediaType) assertEquals(partitions.getPartition().get(0).getLocation(), xp.getLocation()); assertEquals(partitions.getPartition().get(1).getLocation(), xp.getLocation()); assertTrue(partitions.getPartition().get(0).getTimePartitionSpec() == null || partitions.getPartition().get(1).getTimePartitionSpec() == null); XPartition postedPartition, latestPartition; if (partitions.getPartition().get(0).getTimePartitionSpec() == null) { postedPartition = partitions.getPartition().get(1); latestPartition = partitions.getPartition().get(0); assertEquals(postedPartition.getTimePartitionSpec(), xp.getTimePartitionSpec()); assertEquals(postedPartition.getNonTimePartitionSpec(), xp.getNonTimePartitionSpec()); assertNotNull(postedPartition.getFullPartitionSpec()); XTimePartSpecElement timePartSpec = postedPartition.getTimePartitionSpec().getPartSpecElement().iterator().next(); XPartSpecElement fullPartSpec = postedPartition.getFullPartitionSpec().getPartSpecElement().iterator().next(); assertEquals(timePartSpec.getKey(), fullPartSpec.getKey()); assertEquals(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).format(JAXBUtils.getDateFromXML( timePartSpec.getValue())), fullPartSpec.getValue()); assertNull(latestPartition.getTimePartitionSpec()); assertNull(latestPartition.getNonTimePartitionSpec()); assertEquals(latestPartition.getFullPartitionSpec().getPartSpecElement().get(0).getValue(), "latest");
XPartition xp = new XPartition(); xp.setFactOrDimensionTableName("dim_table"); xp.setLocation(new File("target/test-classes/dim2-part").getAbsolutePath()); xp.setUpdatePeriod(XUpdatePeriod.HOURLY); XTimePartSpec timePart = new XTimePartSpec(); XTimePartSpecElement partElement = new XTimePartSpecElement(); partElement.setValue(DatatypeFactory.newInstance().newXMLGregorianCalendar(new GregorianCalendar())); timePart.getPartSpecElement().add(partElement); xp.setTimePartitionSpec(timePart); result = client.addPartitionToDim("dim_table", "local", xp); assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
public static void updatePartitionFromXPartition(Partition partition, XPartition xp) throws ClassNotFoundException { partition.getParameters().putAll(mapFromXProperties(xp.getPartitionParameters())); partition.getTPartition().getSd().getSerdeInfo().setParameters(mapFromXProperties(xp.getSerdeParameters())); partition.setLocation(xp.getLocation()); if (xp.getInputFormat() != null) { partition.setInputFormatClass(Class.forName(xp.getInputFormat()).asSubclass(InputFormat.class)); } if (xp.getOutputFormat() != null) { Class<? extends HiveOutputFormat> outputFormatClass = Class.forName(xp.getOutputFormat()).asSubclass(HiveOutputFormat.class); partition.setOutputFormatClass(outputFormatClass); } partition.getParameters().put(MetastoreConstants.PARTITION_UPDATE_PERIOD, xp.getUpdatePeriod().name()); partition.getTPartition().getSd().getSerdeInfo().setSerializationLib(xp.getSerdeClassname()); }