@Override public Set<DataSegment> findSegments(String workingDirPath, boolean updateDescriptor) throws SegmentLoadingException { final Map<SegmentId, Pair<DataSegment, Long>> timestampedSegments = new HashMap<>(); final File workingDir = new File(workingDirPath); if (!workingDir.isDirectory()) { throw new SegmentLoadingException("Working directory [%s] didn't exist !?", workingDir); } recursiveSearchSegments(timestampedSegments, workingDir, updateDescriptor); return timestampedSegments.values().stream().map(x -> x.lhs).collect(Collectors.toSet()); }
private DataSegmentMover getMover(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentMover mover = movers.get(type); if (mover == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, movers.keySet()); } return mover; } }
private void loadInLocationWithStartMarker(DataSegment segment, File storageDir) throws SegmentLoadingException { // We use a marker to prevent the case where a segment is downloaded, but before the download completes, // the parent directories of the segment are removed final File downloadStartMarker = new File(storageDir, "downloadStartMarker"); synchronized (lock) { if (!storageDir.mkdirs()) { log.debug("Unable to make parent file[%s]", storageDir); } try { if (!downloadStartMarker.createNewFile()) { throw new SegmentLoadingException("Was not able to create new download marker for [%s]", storageDir); } } catch (IOException e) { throw new SegmentLoadingException(e, "Unable to create marker file for [%s]", storageDir); } } loadInLocation(segment, storageDir); if (!downloadStartMarker.delete()) { throw new SegmentLoadingException("Unable to remove marker file for [%s]", storageDir); } }
private DataSegmentArchiver getArchiver(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentArchiver archiver = archivers.get(type); if (archiver == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, archivers.keySet()); } return archiver; } }
private File getFile(DataSegment segment) throws SegmentLoadingException { final Map<String, Object> loadSpec = segment.getLoadSpec(); final File path = new File(MapUtils.getString(loadSpec, "path")); if (!path.exists()) { throw new SegmentLoadingException("Asked to load path[%s], but it doesn't exist.", path); } return path; } }
private DataSegmentKiller getKiller(DataSegment segment) throws SegmentLoadingException { String type = MapUtils.getString(segment.getLoadSpec(), "type"); DataSegmentKiller loader = killers.get(type); if (loader == null) { throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, killers.keySet()); } return loader; }
private boolean isObjectInBucket(final S3Coords coords) throws SegmentLoadingException { try { return S3Utils.retryS3Operation( () -> S3Utils.isObjectInBucketIgnoringPermission(s3Client, coords.bucket, coords.path) ); } catch (AmazonS3Exception | IOException e) { throw new SegmentLoadingException(e, "S3 fail! Key[%s]", coords); } catch (Exception e) { throw Throwables.propagate(e); } }
@Override public Segment getSegment(DataSegment segment) throws SegmentLoadingException { File segmentFiles = getSegmentFiles(segment); File factoryJson = new File(segmentFiles, "factory.json"); final SegmentizerFactory factory; if (factoryJson.exists()) { try { factory = jsonMapper.readValue(factoryJson, SegmentizerFactory.class); } catch (IOException e) { throw new SegmentLoadingException(e, "%s", e.getMessage()); } } else { factory = new MMappedQueryableSegmentizerFactory(indexIO); } return factory.factorize(segment, segmentFiles); }
@Override public Segment factorize(DataSegment dataSegment, File parentDir) throws SegmentLoadingException { try { return new QueryableIndexSegment(indexIO.loadIndex(parentDir), dataSegment.getId()); } catch (IOException e) { throw new SegmentLoadingException(e, "%s", e.getMessage()); } } }
@Override public void kill(DataSegment segment) throws SegmentLoadingException { final File path = getPath(segment); log.info("killing segment[%s] mapped to path[%s]", segment.getId(), path); try { if (path.getName().endsWith(".zip")) { // path format -- > .../dataSource/interval/version/partitionNum/xxx.zip // or .../dataSource/interval/version/partitionNum/UUID/xxx.zip File parentDir = path.getParentFile(); FileUtils.deleteDirectory(parentDir); // possibly recursively delete empty parent directories up to 'dataSource' parentDir = parentDir.getParentFile(); int maxDepth = 4; // if for some reason there's no datasSource directory, stop recursing somewhere reasonable while (parentDir != null && --maxDepth >= 0) { if (!parentDir.delete() || segment.getDataSource().equals(parentDir.getName())) { break; } parentDir = parentDir.getParentFile(); } } else { throw new SegmentLoadingException("Unknown file type[%s]", path); } } catch (IOException e) { throw new SegmentLoadingException(e, "Unable to kill segment"); } }
throw new SegmentLoadingException(e, "Failed to announce segments[%s]", queue); throw new SegmentLoadingException(e, "Loading Interrupted"); throw new SegmentLoadingException(e.getCause(), "Background Announcing Task Failed");
@Override public void run() { synchronized (lock) { try { if (!(finished && queue.isEmpty())) { final List<DataSegment> segments = new ArrayList<>(); queue.drainTo(segments); try { announcer.announceSegments(segments); nextAnnoucement = exec.schedule(this, intervalMillis, TimeUnit.MILLISECONDS); } catch (IOException e) { doneAnnouncing.setException( new SegmentLoadingException(e, "Failed to announce segments[%s]", segments) ); } } else { doneAnnouncing.set(true); } } catch (Exception e) { doneAnnouncing.setException(e); } } } },
final FileSystem fs = segmentPath.getFileSystem(config); if (!filename.endsWith(".zip")) { throw new SegmentLoadingException("Unknown file type[%s]", segmentPath); } else { throw new SegmentLoadingException("Unable to kill segment, failed to delete [%s]", segmentPath.toString()); throw new SegmentLoadingException("Unable to kill segment, failed to delete [%s]", descriptorPath.toString()); throw new SegmentLoadingException(e, "Unable to kill segment");
throw new SegmentLoadingException( e, "Failed to read descriptor.json for segment located at [%s]", throw new SegmentLoadingException( "index.zip didn't exist at [%s] while descripter.json exists!?", indexZip.getAbsoluteFile()
private Segment getAdapter(final DataSegment segment) throws SegmentLoadingException { final Segment adapter; try { adapter = segmentLoader.getSegment(segment); } catch (SegmentLoadingException e) { segmentLoader.cleanup(segment); throw e; } if (adapter == null) { throw new SegmentLoadingException("Null adapter from loadSpec[%s]", segment.getLoadSpec()); } return adapter; }
@Override public void run() { try { log.info( "Loading segment[%d/%d][%s]", counter.incrementAndGet(), numSegments, segment.getId() ); loadSegment(segment, callback); try { backgroundSegmentAnnouncer.announceSegment(segment); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new SegmentLoadingException(e, "Loading Interrupted"); } } catch (SegmentLoadingException e) { log.error(e, "[%s] failed to load", segment.getId()); failedSegments.add(segment); } finally { latch.countDown(); } } }
throw new SegmentLoadingException("Working directory [%s] doesn't exist.", workingDirPath); throw new SegmentLoadingException("Working directory [%s] is not a directory!?", workingDirPath); ); } else { throw new SegmentLoadingException( "index.zip didn't exist at [%s] while descripter.json exists!?", indexZip throw new SegmentLoadingException(e, "Problems interacting with filesystem[%s].", workingDirPath);
); throw new SegmentLoadingException(e, e.getMessage());
); throw new SegmentLoadingException(e, e.getMessage());
@Override public void kill(DataSegment segment) throws SegmentLoadingException { LOG.info("Killing segment [%s]", segment); Map<String, Object> loadSpec = segment.getLoadSpec(); final String bucket = MapUtils.getString(loadSpec, "bucket"); final String indexPath = MapUtils.getString(loadSpec, "path"); final String descriptorPath = indexPath.substring(0, indexPath.lastIndexOf('/')) + "/descriptor.json"; try { deleteIfPresent(bucket, indexPath); deleteIfPresent(bucket, descriptorPath); } catch (IOException e) { throw new SegmentLoadingException(e, "Couldn't kill segment[%s]: [%s]", segment.getId(), e.getMessage()); } }