@Override public Long apply(DatasetDependency dependency) { // first account for the dataset's refresh period final AccelerationSettings settings = reflectionSettings.getReflectionSettings(dependency.getNamespaceKey()); final long refreshStart = Boolean.TRUE.equals(settings.getNeverRefresh()) || settings.getRefreshPeriod() == 0 ? 0 : currentTime - settings.getRefreshPeriod(); // then account for any refresh request against the dataset final RefreshRequest request = requestsStore.get(dependency.getId()); if (request != null) { return Math.max(refreshStart, request.getRequestedAt()); } return refreshStart; } })
public RefreshSettings(AccelerationSettings settings) { refreshField = settings.getRefreshField(); method = settings.getMethod(); refreshPeriodMs = settings.getRefreshPeriod(); gracePeriodMs = settings.getGracePeriod(); }
.setAccelerationRefreshPeriod(settings.getRefreshPeriod()) .setAccelerationGracePeriod(settings.getGracePeriod()) .setMethod(settings.getMethod())
public void setReflectionSettings(NamespaceKey key, AccelerationSettings settings) { // if some settings already exist just override them, otherwise remove the version as the passed settings may be // coming from the parent source AccelerationSettings previous = store.get(key); settings.setTag(previous != null ? previous.getTag() : null); // version is deprecated but may exist after an upgrade so we need to ensure that it is nulled out when we null out // the tag or else the inline upgrade code for OCC will get confused settings.setVersion(previous != null ? previous.getVersion() : null); if (settings.getRefreshPeriod() == null) { settings.setRefreshPeriod(DEFAULT_REFRESH_PERIOD); } if (settings.getGracePeriod() == null) { settings.setGracePeriod(DEFAULT_GRACE_PERIOD); } store.save(key, settings); }
@GET @Produces(MediaType.APPLICATION_JSON) public SourceUI getSource(@QueryParam("includeContents") @DefaultValue("true") boolean includeContents) throws Exception { try { final SourceConfig config = namespaceService.get().getSource(sourcePath.toNamespaceKey()); final SourceState sourceState = sourceService.getSourceState(sourcePath.getSourceName().getName()); if (sourceState == null) { throw new SourceNotFoundException(sourcePath.getSourceName().getName()); } final BoundedDatasetCount datasetCount = namespaceService.get().getDatasetCount(new NamespaceKey(config.getName()), BoundedDatasetCount.SEARCH_TIME_LIMIT_MS, BoundedDatasetCount.COUNT_LIMIT_TO_STOP_SEARCH); final SourceUI source = newSource(config) .setNumberOfDatasets(datasetCount.getCount()); source.setDatasetCountBounded(datasetCount.isCountBound() || datasetCount.isTimeBound()); source.setState(sourceState); final AccelerationSettings settings = reflectionService.get().getReflectionSettings().getReflectionSettings(sourcePath.toNamespaceKey()); if (settings != null) { source.setAccelerationRefreshPeriod(settings.getRefreshPeriod()); source.setAccelerationGracePeriod(settings.getGracePeriod()); } if (includeContents) { source.setContents(sourceService.listSource(sourcePath.getSourceName(), namespaceService.get().getSource(sourcePath.toNamespaceKey()), securityContext.getUserPrincipal().getName())); } return source; } catch (NamespaceNotFoundException nfe) { throw new SourceNotFoundException(sourcePath.getSourceName().getName(), nfe); } }
public Source(SourceConfig config, AccelerationSettings settings, ConnectionReader reader) { this.id = config.getId().getId(); this.tag = config.getTag(); this.type = config.getType() == null ? config.getLegacySourceTypeEnum().name() : config.getType(); this.name = config.getName(); this.description = config.getDescription(); if (config.getCtime() != null) { this.createdAt = config.getCtime(); } com.dremio.service.namespace.source.proto.MetadataPolicy configMetadataPolicy = config.getMetadataPolicy(); if (configMetadataPolicy == null) { this.metadataPolicy = new MetadataPolicy(CatalogService.DEFAULT_METADATA_POLICY); } else { this.metadataPolicy = new MetadataPolicy(configMetadataPolicy); } validator.validate(this.metadataPolicy); this.accelerationGracePeriodMs = settings.getGracePeriod(); this.accelerationRefreshPeriodMs = settings.getRefreshPeriod(); this.accelerationNeverExpire = settings.getNeverExpire(); this.accelerationNeverRefresh = settings.getNeverRefresh(); // TODO: use our own config classes this.config = reader.getConnectionConf(config); }