@Test public void testConstraintsGetReadyTimestamp() throws IOException { SignalManager manager = new SignalManager(fileSystem, testDirectory); Constraints constraints = new Constraints(DatasetTestUtilities.USER_SCHEMA). with("email", "testConstraintsReady@domain.com"); Path signalFilePath = new Path(this.testDirectory, "email=testConstraintsReady%40domain.com"); // drop a file at the signal path FSDataOutputStream stream = this.fileSystem.create(signalFilePath, true); stream.writeUTF(String.valueOf(System.currentTimeMillis())); stream.close(); Assert.assertTrue(manager.getReadyTimestamp(constraints) != -1); }
FileSystemDataset(FileSystem fileSystem, Path directory, String namespace, String name, DatasetDescriptor descriptor, URI uri, @Nullable PartitionListener partitionListener, Class<E> type) { super(type, descriptor.getSchema()); if (Formats.PARQUET.equals(descriptor.getFormat())) { Preconditions.checkArgument(IndexedRecord.class.isAssignableFrom(type) || type == Object.class, "Parquet only supports generic and specific data models, type" + " parameter must implement IndexedRecord"); } this.fileSystem = fileSystem; this.directory = directory; this.namespace = namespace; this.name = name; this.descriptor = descriptor; this.partitionStrategy = descriptor.isPartitioned() ? descriptor.getPartitionStrategy() : null; this.partitionListener = partitionListener; this.convert = new PathConversion(descriptor.getSchema()); this.uri = uri; Path signalsPath = new Path(getDirectory(fileSystem, directory), SIGNALS_DIRECTORY_NAME); this.signalManager = new SignalManager(fileSystem, signalsPath); this.unbounded = new FileSystemPartitionView<E>( this, partitionListener, signalManager, type); // remove this.partitionKey for 0.14.0 this.partitionKey = null; }
@Test public void testSignalDirectoryCreatedOnSignal() throws IOException { SignalManager manager = new SignalManager(fileSystem, testDirectory); Assert.assertFalse("Signal directory shouldn't exist before signals", fileSystem.exists(testDirectory)); Constraints constraints = new Constraints(DatasetTestUtilities.USER_SCHEMA). with("email", "signalCreatesDir@domain.com"); manager.signalReady(constraints); Assert.assertTrue("Signal directory created on signals", fileSystem.exists(testDirectory)); }
@Test public void testConstraintsSignaledReady() throws IOException { SignalManager manager = new SignalManager(fileSystem, testDirectory); Constraints constraints = new Constraints(DatasetTestUtilities.USER_SCHEMA). with("email", "testConstraintsSignaledReady@domain.com"); manager.signalReady(constraints); String normalizedConstraints = SignalManager.getNormalizedConstraints(constraints); Assert.assertTrue(this.fileSystem.exists(new Path(this.testDirectory, normalizedConstraints))); }
@Test public void testMultiConstraintsSignaledReady() throws IOException { SignalManager manager = new SignalManager(fileSystem, testDirectory); Constraints constraints = new Constraints(DatasetTestUtilities.USER_SCHEMA). with("email", "kiteuser@domain.com").with("username", "kiteuser"); manager.signalReady(constraints); String normalizedConstraints = SignalManager.getNormalizedConstraints(constraints); Assert.assertTrue(this.fileSystem.exists(new Path(this.testDirectory, normalizedConstraints))); }
@Test public void testConstraintsSignaledReadyPreviouslySignaled() throws IOException, InterruptedException { SignalManager manager = new SignalManager(fileSystem, testDirectory); Constraints constraints = new Constraints(DatasetTestUtilities.USER_SCHEMA) .with("email", "testConstraintsSignaledReadyPreviouslySignaled@domain.com"); String normalizedConstraints = SignalManager.getNormalizedConstraints(constraints); Path signalFilePath = new Path(this.testDirectory,normalizedConstraints); manager.signalReady(constraints); Assert.assertTrue(this.fileSystem.exists(signalFilePath)); long firstSignalContents = this.fileSystem.getFileStatus(signalFilePath).getModificationTime(); // the modified time depends on the filesystem, and may only be granular to the second // signal and check until the modified time is after the current time, or until // enough time has past that the signal should have been distinguishable long spinStart = System.currentTimeMillis(); long signaledTime = 0; while(firstSignalContents >= signaledTime && (System.currentTimeMillis() - spinStart) <= 2000) { manager.signalReady(constraints); signaledTime = manager.getReadyTimestamp(constraints); } Assert.assertFalse("Second signal should not match the first", signaledTime == firstSignalContents); }
@Test public void testConstraintsGetReadyTimestampNotYetSignaled() throws IOException { SignalManager manager = new SignalManager(fileSystem, testDirectory); Constraints constraints = new Constraints(DatasetTestUtilities.USER_SCHEMA). with("email", "testConstraintsGetReadyTimestampNotYetSignaled@domain.com"); Assert.assertEquals("A constraint that is not signaled should show -1", -1, manager.getReadyTimestamp(constraints)); }