private static Collection<XExprSpec> xExprSpecFromExprColumn(Collection<ExprSpec> esSet) { List<XExprSpec> xes = new ArrayList<XExprSpec>(); for (ExprSpec es : esSet) { XExprSpec e = new XExprSpec(); e.setExpr(es.getExpr()); e.setStartTime(getXMLGregorianCalendar(es.getStartTime())); e.setEndTime(getXMLGregorianCalendar(es.getEndTime())); xes.add(e); } return xes; }
private void addAllMeasuresToFlattenedList(ObjectFactory objectFactory, CubeInterface cube, List<XFlattenedColumn> columnList) { for (CubeMeasure msr : cube.getMeasures()) { XFlattenedColumn fcol = objectFactory.createXFlattenedColumn(); fcol.setMeasure(JAXBUtils.xMeasureFromHiveMeasure(msr)); fcol.setTableName(cube.getName()); columnList.add(fcol); } }
public void alterCubeDimensionTable(XDimensionTable dimensionTable) throws LensException, HiveException { alterCubeDimensionTable(dimensionTable.getTableName(), JAXBUtils.cubeDimTableFromDimTable(dimensionTable), JAXBUtils.tableDescPrefixMapFromXStorageTables(dimensionTable.getStorageTables())); } /**
public void alterCubeFactTable(XFact fact) throws LensException, HiveException { if (fact instanceof XVirtualFactTable) { XVirtualFactTable xvf = (XVirtualFactTable) fact; alterCubeFactTable(xvf.getName(), JAXBUtils.cubeVirtualFactFromFactTable(xvf, getFactTable(xvf.getSourceFactName())), null, new HashMap<>()); } else { XFactTable xf = (XFactTable) fact; alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(xf), JAXBUtils.tableDescPrefixMapFromXStorageTables(xf.getStorageTables()), JAXBUtils.columnStartAndEndTimeFromXColumns(xf.getColumns())); } }
public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); if (!dimTable.getStorages().isEmpty()) { for (String storageName : dimTable.getStorages()) { XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } dt.getStorageTables().getStorageTable().add(tblElement); } } return dt; }
public static XDimension xdimensionFromDimension(Dimension dimension) { XDimension xd = XCF.createXDimension(); xd.setName(dimension.getName()); xd.setAttributes(new XDimAttributes()); xd.setExpressions(new XExpressions()); xd.setJoinChains(new XJoinChains()); xd.setProperties(new XProperties()); xd.getProperties().getProperty().addAll(xPropertiesFromMap(((AbstractCubeTable) dimension).getProperties())); for (CubeDimAttribute cd : dimension.getAttributes()) { xd.getAttributes().getDimAttribute().add(xDimAttrFromHiveDimAttr(cd, dimension)); } for (ExprColumn ec : dimension.getExpressions()) { xd.getExpressions().getExpression().add(xExprColumnFromHiveExprColumn(ec)); } for (JoinChain jc : dimension.getJoinChains()) { xd.getJoinChains().getJoinChain().add(getXJoinChainFromJoinChain(jc)); } return xd; }
xd.setDescription(cd.getDescription()); xd.setDisplayString(cd.getDisplayString()); xd.setStartTime(getXMLGregorianCalendar(cd.getStartTime())); xd.setEndTime(getXMLGregorianCalendar(cd.getEndTime())); xd.setTags(getXProperties(xPropertiesFromMap(cd.getTags()))); if (cd instanceof ReferencedDimAttribute) { ReferencedDimAttribute rd = (ReferencedDimAttribute) cd; XDimAttributes hierarchy = new XDimAttributes(); for (CubeDimAttribute hdDim : hd.getHierarchy()) { hierarchy.getDimAttribute().add(xDimAttrFromHiveDimAttr(hdDim, baseTable));
if (ft.isVirtualFact()) { CubeVirtualFactTable cvft = (CubeVirtualFactTable) ft; XVirtualFactTable factTable = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cvft); factTable.setSourceFactName(cvft.getSourceCubeFactTable().getName()); fact = factTable; } else { CubeFactTable cft = (CubeFactTable) ft; XFactTable factTable = JAXBUtils.factTableFromCubeFactTable(cft); Map<String, Map<UpdatePeriod, String>> storageMap = cft.getStoragePrefixUpdatePeriodMap(); for (String storageName : cft.getStorages()) { XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName))); tblElement.setStorageName(storageName); for (Map.Entry entry : updatePeriodToTableMap.entrySet()) { XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor(); updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), (String) entry.getValue()))));
/** * Create XMeasure from hive ql cube measure */ public static XMeasure xMeasureFromHiveMeasure(CubeMeasure cm) { if (cm == null) { return null; } XMeasure xm = XCF.createXMeasure(); xm.setName(cm.getName()); xm.setDescription(cm.getDescription()); xm.setDisplayString(cm.getDisplayString()); xm.setDefaultAggr(cm.getAggregate()); xm.setFormatString(cm.getFormatString()); xm.setType(XMeasureType.valueOf(cm.getType().toUpperCase())); xm.setUnit(cm.getUnit()); xm.setStartTime(getXMLGregorianCalendar(cm.getStartTime())); xm.setEndTime(getXMLGregorianCalendar(cm.getEndTime())); xm.setMin(cm.getMin()); xm.setMax(cm.getMax()); xm.setTags(getXProperties(xPropertiesFromMap(cm.getTags()))); return xm; }
Date startDate = getDateFromXML(xd.getStartTime()); Date endDate = getDateFromXML(xd.getEndTime()); List<CubeDimAttribute> hierarchy = new ArrayList<>(); for (XDimAttribute hd : xd.getHierarchy().getDimAttribute()) { hierarchy.add(hiveDimAttrFromXDimAttr(hd)); xd.getDescription()), xd.getDisplayString(), getChainRefColumns(xd.getChainRefColumn()), startDate, endDate, xd.getNumDistinctValues(), xd.getValues(), mapFromXProperties(xd.getTags()) ); } else { xd.getNumDistinctValues(), xd.getValues(), mapFromXProperties(xd.getTags()) );
xp.setInputFormat(p.getInputFormatClass().getCanonicalName()); xp.setOutputFormat(p.getOutputFormatClass().getCanonicalName()); xp.getPartitionParameters().getProperty().addAll(xPropertiesFromMap(p.getParameters())); String upParam = p.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD); xp.setUpdatePeriod(XUpdatePeriod.valueOf(upParam)); timePartSpecElement.setKey(entry.getKey()); timePartSpecElement .setValue(getXMLGregorianCalendar(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).parse( entry.getValue()))); xp.getTimePartitionSpec().getPartSpecElement().add(timePartSpecElement); xp.getSerdeParameters().getProperty().addAll(xPropertiesFromMap( p.getTPartition().getSd().getSerdeInfo().getParameters())); return xp;
assertTrue(gotFact.getName().equalsIgnoreCase(table)); assertEquals(gotFact.getWeight(), 10.0); CubeVirtualFactTable cvf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, JAXBUtils.cubeFactFromFactTable(source)); XVirtualFactTable update = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cvf); .get(new GenericType<JAXBElement<XFact>>() {}); gotFact = (XVirtualFactTable) gotFactElement.getValue(); CubeVirtualFactTable ucf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, JAXBUtils.cubeFactFromFactTable(source));
assertEquals(updatePeriodTableDescriptor.size(), 3); CubeFactTable cf = JAXBUtils.cubeFactFromFactTable(gotFact); cf.alterColumn(new FieldSchema("c2", "double", "changed to double")); XFactTable update = JAXBUtils.factTableFromCubeFactTable(cf); XStorageTableElement s1Tbl = createStorageTblWithMultipleTableDescriptors("S1", new String[] { tables[0], tables[1] }, new String[] { updatePeriods[0], updatePeriods[1] }); }); gotFact = (XFactTable) gotFactElement.getValue(); CubeFactTable ucf = JAXBUtils.cubeFactFromFactTable(gotFact); assertEquals(ucf.weight(), 20.0); assertTrue(ucf.getUpdatePeriods().get("S1").contains(HOURLY)); assertEquals(timePartSpec.getKey(), fullPartSpec.getKey()); assertEquals( UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).format(JAXBUtils.getDateFromXML(timePartSpec.getValue())), fullPartSpec.getValue()); DateTime date = target().path("metastore/cubes").path("testCube").path("latestdate")
public void testMeasureJaxBConversion() throws Exception { CubeMeasure cubeMeasure = new ColumnMeasure(new FieldSchema("msr1", "int", "first measure"), null, null, null, null, null, null, null, 0.0, 9999.0); XMeasure measure = JAXBUtils.xMeasureFromHiveMeasure(cubeMeasure); CubeMeasure actualMeasure = JAXBUtils.hiveMeasureFromXMeasure(measure); assertEquals(actualMeasure, cubeMeasure); assertEquals(actualMeasure.getMin(), measure.getMin()); assertEquals(actualMeasure.getMax(), measure.getMax()); }
private static ExprSpec[] exprSpecFromXExprColumn(Collection<XExprSpec> xesList) throws LensException { List<ExprSpec> esArray = new ArrayList<ExprSpec>(xesList.size()); for (XExprSpec xes : xesList) { esArray.add(new ExprSpec(xes.getExpr(), getDateFromXML(xes.getStartTime()), getDateFromXML(xes.getEndTime()))); } return esArray.toArray(new ExprSpec[0]); }
assertTrue(gotFact.getName().equalsIgnoreCase(table)); assertEquals(gotFact.getWeight(), 10.0); CubeFactTable cf = JAXBUtils.cubeFactFromFactTable(gotFact); Map<String, String> props = JAXBUtils.mapFromXProperties(gotFact.getProperties()); assertEquals(props.get(MetastoreConstants.FACT_COL_START_TIME_PFX.concat("c3")), "2016-01-01"); assertEquals(props.get(MetastoreConstants.FACT_COL_END_TIME_PFX.concat("c3")), "2017-01-01");
assertTrue(gotFact.getName().equalsIgnoreCase(virtualFactTable)); assertEquals(gotFact.getWeight(), 10.0); CubeVirtualFactTable cvf = JAXBUtils.cubeVirtualFactFromFactTable(gotFact, JAXBUtils.cubeFactFromFactTable(source));
assertTrue(gotFact.getName().equalsIgnoreCase(table)); assertEquals(gotFact.getWeight(), 10.0); CubeFactTable cf = JAXBUtils.cubeFactFromFactTable(gotFact); cf.alterColumn(new FieldSchema("c2", "double", "changed to double")); XFactTable update = JAXBUtils.factTableFromCubeFactTable(cf); XStorageTableElement s1Tbl = createStorageTblElement("S1", table, "HOURLY"); s1Tbl.getTableDesc().setFieldDelimiter("#"); .get(new GenericType<JAXBElement<XFact>>() {}); gotFact = (XFactTable) gotFactElement.getValue(); CubeFactTable ucf = JAXBUtils.cubeFactFromFactTable(gotFact);
assertEquals(testDim.getName(), "testdim"); assertTrue(testDim.getProperties().getProperty().size() >= 1); assertTrue(JAXBUtils.mapFromXProperties(testDim.getProperties()).containsKey("dimension.foo")); assertEquals(JAXBUtils.mapFromXProperties(testDim.getProperties()).get("dimension.foo"), "dim.bar"); assertEquals(testDim.getAttributes().getDimAttribute().size(), 5); assertEquals(testDim.getExpressions().getExpression().size(), 1); assertEquals(testDim.getJoinChains().getJoinChain().get(0).getDisplayString(), "Chain-1"); Dimension dim = JAXBUtils.dimensionFromXDimension(testDim); assertNotNull(dim.getAttributeByName("col1")); assertEquals(dim.getAttributeByName("col1").getDescription(), "first column"); assertEquals(altered.getName(), "testdim"); assertTrue(altered.getProperties().getProperty().size() >= 2); assertTrue(JAXBUtils.mapFromXProperties(altered.getProperties()).containsKey("dim.prop2.name")); assertEquals(JAXBUtils.mapFromXProperties(altered.getProperties()).get("dim.prop2.name"), "dim.prop2.value"); assertTrue(JAXBUtils.mapFromXProperties(altered.getProperties()).containsKey("dimension.foo")); assertEquals(JAXBUtils.mapFromXProperties(altered.getProperties()).get("dimension.foo"), "dim.bar"); assertEquals(testDim.getAttributes().getDimAttribute().size(), 5); dim = JAXBUtils.dimensionFromXDimension(altered); assertNotNull(dim.getAttributeByName("col3Added")); assertTrue(dim.getAttributeByName("col2") == null || dim.getAttributeByName("col1") == null
public void createDimension(XDimension dim) throws LensException { createDimension(JAXBUtils.dimensionFromXDimension(dim)); } /**