private boolean partitionExists( String tableName, Integer partition ) { return tableExists( PartitionUtils.getPartitionName( tableName, partition ) ); }
/** * Returns the end date for the given year, exclusive, i.e. * the start date of the year after the given year. * * @param calendar the calendar to base the date on. * @param year the year. * @return the start date. */ public static Date getEndDate( Calendar calendar, Integer year ) { Integer nextYear = year + 1; return getStartDate( calendar, nextYear ); }
/** * Analyzes the given analytics tables. * * @param tables the list of {@link AnalyticsTable}. */ private void analyzeTables( List<AnalyticsTable> tables ) { List<AnalyticsTablePartition> partitions = PartitionUtils.getTablePartitions( tables ); partitions.forEach( table -> tableManager.analyzeTable( table.getTempTableName() ) ); }
/** * Generates a list of {@link AnalyticsTable} based on a list of years with data. * * @param dataYears the list of years with data. * @param dimensionColumns the list of dimension {@link AnalyticsTableColumn}. * @param valueColumns the list of value {@link AnalyticsTableColumn}. */ protected AnalyticsTable getAnalyticsTable( List<Integer> dataYears, List<AnalyticsTableColumn> dimensionColumns, List<AnalyticsTableColumn> valueColumns ) { Calendar calendar = PeriodType.getCalendar(); Collections.sort( dataYears ); String baseName = getAnalyticsTableType().getTableName(); AnalyticsTable table = new AnalyticsTable( baseName, dimensionColumns, valueColumns ); for ( Integer year : dataYears ) { table.addPartitionTable( year, PartitionUtils.getStartDate( calendar, year ), PartitionUtils.getEndDate( calendar, year ) ); } return table; }
/** * Sets table name and partitions on the given query. * * @param params the event query parameters. * @return a {@link EventQueryParams}. */ private EventQueryParams withTableNameAndPartitions( EventQueryParams params ) { Partitions partitions = params.hasStartEndDate() ? PartitionUtils.getPartitions( params.getStartDate(), params.getEndDate() ) : PartitionUtils.getPartitions( params.getAllPeriods() ); String baseName = params.hasEnrollmentProgramIndicatorDimension() ? AnalyticsTableType.ENROLLMENT.getTableName() : AnalyticsTableType.EVENT.getTableName(); String tableName = PartitionUtils.getTableName( baseName, params.getProgram() ); return new EventQueryParams.Builder( params ) .withTableName( tableName ) .withPartitions( partitions ) .build(); }
/** * Returns partitions for the given start and end date. * * @param startDate the start date. * @param endDate the end date. * @return partitions for the given start and end date. */ public static Partitions getPartitions( Date startDate, Date endDate ) { Period period = new Period(); period.setStartDate( startDate ); period.setEndDate( endDate ); return getPartitions( period ); }
/** * Returns partitions for the given period. * * @param period the period. * @return partitions for the given period. */ public static Partitions getPartitions( Period period ) { return new Partitions( getYears( period ) ); }
ListMap<String, DimensionalItemObject> periodTypePeriodMap = PartitionUtils.getPeriodTypePeriodMap( params.getPeriods() ); ListMap<String, DimensionalItemObject> periodTypePeriodMap = PartitionUtils.getPeriodTypePeriodMap( filter.getItems() );
@Override @Transactional public List<AnalyticsTable> getAnalyticsTables( Date earliest ) { log.info( String.format( "Get tables using earliest: %s, spatial support: %b", earliest, databaseInfo.isSpatialSupport() ) ); List<AnalyticsTable> tables = new ArrayList<>(); Calendar calendar = PeriodType.getCalendar(); String baseName = getTableName(); List<Program> programs = idObjectManager.getAllNoAcl( Program.class ); for ( Program program : programs ) { List<Integer> dataYears = getDataYears( program, earliest ); Collections.sort( dataYears ); AnalyticsTable table = new AnalyticsTable( baseName, getDimensionColumns( program ), Lists.newArrayList(), program ); for ( Integer year : dataYears ) { table.addPartitionTable( year, PartitionUtils.getStartDate( calendar, year ), PartitionUtils.getEndDate( calendar, year ) ); } if ( table.hasPartitionTables() ) { tables.add( table ); } } return tables; }
@Override public DataQueryParams withTableNameAndPartitions( DataQueryParams params, QueryPlannerParams plannerParams ) { Partitions partitions = params.hasStartEndDate() ? PartitionUtils.getPartitions( params.getStartDate(), params.getEndDate() ) : PartitionUtils.getPartitions( params.getAllPeriods() ); return DataQueryParams.newBuilder( params ) .withTableName( plannerParams.getTableName() ) .withPartitions( partitions ) .build(); }
/** * Returns partitions for the given list of periods. * * @param period the period. * @return partitions for the given list of periods. */ public static Partitions getPartitions( List<DimensionalItemObject> periods ) { final Set<Integer> years = new HashSet<>(); periods.forEach( p -> { Period period = (Period) p; years.addAll( getYears( period ) ); } ); return new Partitions( years ); }
/** * Populates the given analytics tables. * * @param tables the list of {@link AnalyticsTable}. */ private void populateTables( AnalyticsTableUpdateParams params, List<AnalyticsTable> tables ) { List<AnalyticsTablePartition> partitions = PartitionUtils.getTablePartitions( tables ); int taskNo = Math.min( getProcessNo(), partitions.size() ); log.info( "Populate table task number: " + taskNo ); ConcurrentLinkedQueue<AnalyticsTablePartition> partitionQ = new ConcurrentLinkedQueue<>( partitions ); List<Future<?>> futures = new ArrayList<>(); for ( int i = 0; i < taskNo; i++ ) { futures.add( tableManager.populateTablesAsync( params, partitionQ ) ); } ConcurrentUtils.waitForCompletion( futures ); }
final Date earliest = PartitionUtils.getStartDate( params.getLastYears() );
/** * Returns the query from source clause. Can be any of table name, partition * name or inner select union all query. */ private String getFromSourceClause( DataQueryParams params ) { if ( !params.isSkipPartitioning() && params.hasPartitions() && params.getPartitions().hasOne() ) { Integer partition = params.getPartitions().getAny(); return PartitionUtils.getPartitionName( params.getTableName(), partition ); } else if ( ( !params.isSkipPartitioning() && params.hasPartitions() && params.getPartitions().hasMultiple() ) ) { String sql = "("; for ( Integer partition : params.getPartitions().getPartitions() ) { String partitionName = PartitionUtils.getPartitionName( params.getTableName(), partition ); sql += "select ap.* from " + partitionName + " as ap union all "; } return TextUtils.removeLast( sql, "union all" ) + ")"; } else { return params.getTableName(); } }
/** * Vacuums the given analytics tables. * * @param tables the list of {@link AnalyticsTable}. */ private void vacuumTables( List<AnalyticsTable> tables ) { List<AnalyticsTablePartition> partitions = PartitionUtils.getTablePartitions( tables ); ConcurrentLinkedQueue<AnalyticsTablePartition> partitionQ = new ConcurrentLinkedQueue<>( partitions ); List<Future<?>> futures = new ArrayList<>(); for ( int i = 0; i < getProcessNo(); i++ ) { tableManager.vacuumTablesAsync( partitionQ ); } ConcurrentUtils.waitForCompletion( futures ); }
List<AnalyticsTablePartition> partitions = PartitionUtils.getTablePartitions( tables );
/** * Creates indexes on the given analytics tables. * * @param tables the list of {@link AnalyticsTable}. */ private void createIndexes( List<AnalyticsTable> tables ) { List<AnalyticsTablePartition> partitions = PartitionUtils.getTablePartitions( tables ); ConcurrentLinkedQueue<AnalyticsIndex> indexes = new ConcurrentLinkedQueue<>(); for ( AnalyticsTablePartition partition : partitions ) { List<AnalyticsTableColumn> columns = partition.getMasterTable().getDimensionColumns(); for ( AnalyticsTableColumn col : columns ) { if ( !col.isSkipIndex() ) { List<String> indexColumns = col.hasIndexColumns() ? col.getIndexColumns() : Lists.newArrayList( col.getName() ); indexes.add( new AnalyticsIndex( partition.getTempTableName(), indexColumns, col.getIndexType() ) ); } } } log.info( "No of analytics table indexes: " + indexes.size() ); List<Future<?>> futures = new ArrayList<>(); for ( int i = 0; i < getProcessNo(); i++ ) { futures.add( tableManager.createIndexesAsync( indexes ) ); } ConcurrentUtils.waitForCompletion( futures ); }