@Override public Double asDataSize(final DataUnit dataUnit) { return rawValue == null ? null : DataUnit.parseDataSize(rawValue.trim(), dataUnit); }
public VolatileContentRepository(final NiFiProperties nifiProperties) { final String maxSize = nifiProperties.getProperty(MAX_SIZE_PROPERTY); final String blockSizeVal = nifiProperties.getProperty(BLOCK_SIZE_PROPERTY); if (maxSize == null) { maxBytes = (long) DataUnit.B.convert(100D, DataUnit.MB); } else { maxBytes = DataUnit.parseDataSize(maxSize, DataUnit.B).longValue(); } final int blockSize; if (blockSizeVal == null) { blockSize = (int) DataUnit.B.convert(DEFAULT_BLOCK_SIZE_KB, DataUnit.KB); } else { blockSize = DataUnit.parseDataSize(blockSizeVal, DataUnit.B).intValue(); } memoryManager = new MemoryManager(maxBytes, blockSize); }
@Override public double convert(double sourceSize, DataUnit sourceUnit) { return sourceUnit.toB(sourceSize); } },
public static Double parseDataSize(final String value, final DataUnit units) { if (value == null) { return null; } final Matcher matcher = DATA_SIZE_PATTERN.matcher(value.toUpperCase()); if (!matcher.find()) { throw new IllegalArgumentException("Invalid data size: " + value); } final String sizeValue = matcher.group(1); final String unitValue = matcher.group(2); final DataUnit sourceUnit = DataUnit.valueOf(unitValue); final double size = Double.parseDouble(sizeValue); return units.convert(size, sourceUnit); } }
final double maxBytes = DataUnit.B.convert(maxSize, unit);
@Override public double convert(double sourceSize, DataUnit sourceUnit) { return sourceUnit.toGB(sourceSize); } },
final double maxBytes = DataUnit.B.convert(maxSize, unit);
public void setBackPressureDataSizeThreshold(String backPressureDataSizeThreshold) { this.backPressureDataSizeThreshold = backPressureDataSizeThreshold; setBackPressureBytesThreshold(DataUnit.parseDataSize(backPressureDataSizeThreshold, DataUnit.B).longValue()); }
@Override public void setBackPressureDataSizeThreshold(final String maxDataSize) { final long maxBytes = DataUnit.parseDataSize(maxDataSize, DataUnit.B).longValue(); boolean updated = false; while (!updated) { MaxQueueSize maxSize = getMaxQueueSize(); final MaxQueueSize updatedSize = new MaxQueueSize(maxDataSize, maxBytes, maxSize.getMaxCount()); updated = maxQueueSize.compareAndSet(maxSize, updatedSize); } }
@Override public void onPropertyModified(final PropertyDescriptor descriptor, final String oldValue, final String newValue) { super.onPropertyModified(descriptor, oldValue, newValue); if (descriptor.equals(RATE_CONTROL_CRITERIA) || descriptor.equals(RATE_CONTROL_ATTRIBUTE_NAME) || descriptor.equals(GROUPING_ATTRIBUTE_NAME) || descriptor.equals(TIME_PERIOD)) { // if the criteria that is being used to determine limits/throttles is changed, we must clear our throttle map. throttleMap.clear(); } else if (descriptor.equals(MAX_RATE)) { final long newRate; if (DataUnit.DATA_SIZE_PATTERN.matcher(newValue.toUpperCase()).matches()) { newRate = DataUnit.parseDataSize(newValue, DataUnit.B).longValue(); } else { newRate = Long.parseLong(newValue); } for (final Throttle throttle : throttleMap.values()) { throttle.setMaxRate(newRate); } } }
public FlowConfigurationArchiveManager(final Path flowConfigFile, final NiFiProperties properties) { final String archiveDirVal = properties.getFlowConfigurationArchiveDir(); final Path archiveDir = (archiveDirVal == null || archiveDirVal.equals("")) ? flowConfigFile.getParent().resolve("archive") : new File(archiveDirVal).toPath(); this.maxCount = properties.getFlowConfigurationArchiveMaxCount(); String maxTime = properties.getFlowConfigurationArchiveMaxTime(); String maxStorage = properties.getFlowConfigurationArchiveMaxStorage(); if (maxCount == null && StringUtils.isBlank(maxTime) && StringUtils.isBlank(maxStorage)) { // None of limitation is specified, fall back to the default configuration; maxTime = NiFiProperties.DEFAULT_FLOW_CONFIGURATION_ARCHIVE_MAX_TIME; maxStorage = NiFiProperties.DEFAULT_FLOW_CONFIGURATION_ARCHIVE_MAX_STORAGE; logger.info("None of archive max limitation is specified, fall back to the default configuration, maxTime={}, maxStorage={}", maxTime, maxStorage); } this.maxTimeMillis = StringUtils.isBlank(maxTime) ? null : FormatUtils.getTimeDuration(maxTime, TimeUnit.MILLISECONDS); this.maxStorageBytes = StringUtils.isBlank(maxStorage) ? null : DataUnit.parseDataSize(maxStorage, DataUnit.B).longValue(); this.flowConfigFile = flowConfigFile; this.archiveDir = archiveDir; }
final long configuredAppendableClaimLength = DataUnit.parseDataSize(nifiProperties.getMaxAppendableClaimSize(), DataUnit.B).longValue(); final long appendableClaimLengthCap = DataUnit.parseDataSize(APPENDABLE_CLAIM_LENGTH_CAP, DataUnit.B).longValue(); if (configuredAppendableClaimLength > appendableClaimLengthCap) { LOG.warn("Configured property '{}' with value {} exceeds cap of {}. Setting value to {}",
long calculatedThreshold; if (DATA_SIZE_PATTERN.matcher(thresholdValue).matches()) { calculatedThreshold = DataUnit.parseDataSize(thresholdValue, DataUnit.B).longValue(); } else { final String percentage = thresholdValue.substring(0, thresholdValue.length() - 1);
private void configureConnectors(final Server server) throws ServerConfigurationException { // create the http configuration final HttpConfiguration httpConfiguration = new HttpConfiguration(); final int headerSize = DataUnit.parseDataSize(props.getWebMaxHeaderSize(), DataUnit.B).intValue(); httpConfiguration.setRequestHeaderSize(headerSize); httpConfiguration.setResponseHeaderSize(headerSize); // Check if both HTTP and HTTPS connectors are configured and fail if both are configured if (bothHttpAndHttpsConnectorsConfigured(props)) { logger.error("NiFi only supports one mode of HTTP or HTTPS operation, not both simultaneously. " + "Check the nifi.properties file and ensure that either the HTTP hostname and port or the HTTPS hostname and port are empty"); startUpFailure(new IllegalStateException("Only one of the HTTP and HTTPS connectors can be configured at one time")); } if (props.getSslPort() != null) { configureHttpsConnector(server, httpConfiguration); } else if (props.getPort() != null) { configureHttpConnector(server, httpConfiguration); } else { logger.error("Neither the HTTP nor HTTPS connector was configured in nifi.properties"); startUpFailure(new IllegalStateException("Must configure HTTP or HTTPS connector")); } }
newRate = DataUnit.parseDataSize(maximumRateStr, DataUnit.B).longValue(); } else { newRate = Long.parseLong(maximumRateStr);
final long maxFileSize = DataUnit.parseDataSize(query.getMaxFileSize(), DataUnit.B).longValue(); if (event.getFileSize() > maxFileSize) { return false; final long minFileSize = DataUnit.parseDataSize(query.getMinFileSize(), DataUnit.B).longValue(); if (event.getFileSize() < minFileSize) { return false;
public static org.apache.lucene.search.Query convertQuery(final org.apache.nifi.provenance.search.Query query) { if (query.getStartDate() == null && query.getEndDate() == null && query.getSearchTerms().isEmpty()) { return new MatchAllDocsQuery(); } final BooleanQuery luceneQuery = new BooleanQuery(); for (final SearchTerm searchTerm : query.getSearchTerms()) { final String searchValue = searchTerm.getValue(); if (searchValue == null) { throw new IllegalArgumentException("Empty search value not allowed (for term '" + searchTerm.getSearchableField().getFriendlyName() + "')"); } if (searchValue.contains("*") || searchValue.contains("?")) { luceneQuery.add(new BooleanClause(new WildcardQuery(new Term(searchTerm.getSearchableField().getSearchableFieldName(), searchTerm.getValue().toLowerCase())), Occur.MUST)); } else { luceneQuery.add(new BooleanClause(new TermQuery(new Term(searchTerm.getSearchableField().getSearchableFieldName(), searchTerm.getValue().toLowerCase())), Occur.MUST)); } } final Long minBytes = query.getMinFileSize() == null ? null : DataUnit.parseDataSize(query.getMinFileSize(), DataUnit.B).longValue(); final Long maxBytes = query.getMaxFileSize() == null ? null : DataUnit.parseDataSize(query.getMaxFileSize(), DataUnit.B).longValue(); if (minBytes != null || maxBytes != null) { luceneQuery.add(NumericRangeQuery.newLongRange(SearchableFields.FileSize.getSearchableFieldName(), minBytes, maxBytes, true, true), Occur.MUST); } final Long minDateTime = query.getStartDate() == null ? null : query.getStartDate().getTime(); final Long maxDateTime = query.getEndDate() == null ? null : query.getEndDate().getTime(); if (maxDateTime != null || minDateTime != null) { luceneQuery.add(NumericRangeQuery.newLongRange(SearchableFields.EventTime.getSearchableFieldName(), minDateTime, maxDateTime, true, true), Occur.MUST); } return luceneQuery; }
final long maxStorageBytes = DataUnit.parseDataSize(storageSize, DataUnit.B).longValue(); final long rolloverMillis = FormatUtils.getTimeDuration(rolloverTime, TimeUnit.MILLISECONDS); final long rolloverBytes = DataUnit.parseDataSize(rolloverSize, DataUnit.B).longValue(); config.setDesiredIndexSize(DataUnit.parseDataSize(shardSize, DataUnit.B).longValue());
final Double backPressureBytes = DataUnit.parseDataSize(backPressureDataSize, DataUnit.B); if (backPressureBytes > 0) { matches.add("Back pressure data size: " + backPressureDataSize);
@Override public void onSchedulingStart() { super.onSchedulingStart(); final long penalizationMillis = FormatUtils.getTimeDuration(remoteGroup.getYieldDuration(), TimeUnit.MILLISECONDS); final SiteToSiteClient.Builder clientBuilder = new SiteToSiteClient.Builder() .urls(SiteToSiteRestApiClient.parseClusterUrls(remoteGroup.getTargetUris())) .portIdentifier(getTargetIdentifier()) .sslContext(sslContext) .useCompression(isUseCompression()) .eventReporter(remoteGroup.getEventReporter()) .peerPersistenceFile(getPeerPersistenceFile(getIdentifier(), nifiProperties, remoteGroup.getTransportProtocol())) .nodePenalizationPeriod(penalizationMillis, TimeUnit.MILLISECONDS) .timeout(remoteGroup.getCommunicationsTimeout(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS) .transportProtocol(remoteGroup.getTransportProtocol()) .httpProxy(new HttpProxy(remoteGroup.getProxyHost(), remoteGroup.getProxyPort(), remoteGroup.getProxyUser(), remoteGroup.getProxyPassword())) .localAddress(remoteGroup.getLocalAddress()); final Integer batchCount = getBatchCount(); if (batchCount != null) { clientBuilder.requestBatchCount(batchCount); } final String batchSize = getBatchSize(); if (batchSize != null && batchSize.length() > 0) { clientBuilder.requestBatchSize(DataUnit.parseDataSize(batchSize.trim(), DataUnit.B).intValue()); } final String batchDuration = getBatchDuration(); if (batchDuration != null && batchDuration.length() > 0) { clientBuilder.requestBatchDuration(FormatUtils.getTimeDuration(batchDuration.trim(), TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS); } clientRef.set(clientBuilder.build()); }