public static void importNodes( int numRunners, Input input, BatchingNeoStores stores, IdMapper idMapper, ExecutionMonitor executionMonitor, Monitor monitor ) throws IOException { Supplier<EntityImporter> importers = () -> new NodeImporter( stores, idMapper, monitor ); importData( NODE_IMPORT_NAME, numRunners, input.nodes(), stores, importers, executionMonitor, new MemoryUsageStatsProvider( stores, idMapper ) ); }
@Override public void doImport( Input input ) throws IOException { consume( "nodes", input.nodes().iterator(), nodeHeader, RandomEntityDataGenerator::convert ); consume( "relationships", input.relationships().iterator(), relationshipHeader, RandomEntityDataGenerator::convert ); }
@Test public void shouldPropagateExceptionFromFailingDecorator() throws Exception { // GIVEN RuntimeException failure = new RuntimeException( "FAILURE" ); Iterable<DataFactory> data = DataFactories.datas( CsvInputTest.data( ":ID,name\n1,Mattias", new FailingNodeDecorator( failure ) ) ); Input input = new CsvInput( data, defaultFormatNodeFileHeader(), datas(), defaultFormatNodeFileHeader(), IdType.INTEGER, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { readNext( nodes ); } catch ( InputException e ) { // THEN assertSame( e.getCause(), failure ); } }
@Test public void shouldNotParsePointPropertyValuesWithDuplicateKeys() throws Exception { // GIVEN DataFactory data = data( ":ID,name,point:Point\n" + "1,Johan,\" { height :0.01 ,longitude:5, latitude : -4.2, latitude : 4.2 } \"\n" ); Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN readNext( nodes ); fail( "Should have failed when key assigned multiple times, but didn't." ); } catch ( InputException ignore ) { // this is fine } }
@Test public void shouldParsePointPropertyValuesWithCRSInHeader() throws Exception { // GIVEN DataFactory data = data( ":ID,name,point:Point{crs:WGS-84-3D}\n" + "0,Johan,\" { height :0.01 ,longitude:5, latitude : -4.2 } \"\n" ); Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, 0L, new Object[]{"name", "Johan", "point", Values.pointValue( CoordinateReferenceSystem.WGS84_3D, 5, -4.2, 0.01)}, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldUseHeaderInformationToParsePoint() throws Exception { // GIVEN DataFactory data = data( ":ID,name,point:Point{crs:WGS-84}\n" + "0,Johan,\" { x :1 ,y:2 } \"\n" ); Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, 0L, new Object[]{"name", "Johan", "point", Values.pointValue( CoordinateReferenceSystem.WGS84, 1, 2)}, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldIgnoreEmptyIntPropertyValues() throws Exception { // GIVEN DataFactory data = data( ":ID,name,extra:int\n" + "0,Mattias,\n" + // here we leave out "extra" property "1,Johan,10\n" ); Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, 0L, new Object[] {"name", "Mattias"}, labels() ); assertNextNode( nodes, 1L, new Object[] {"name", "Johan", "extra", 10}, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldIgnoreEmptyPropertyValues() throws Exception { // GIVEN DataFactory data = data( ":ID,name,extra\n" + "0,Mattias,\n" + // here we leave out "extra" property "1,Johan,Additional\n" ); Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, 0L, new Object[] {"name", "Mattias"}, labels() ); assertNextNode( nodes, 1L, new Object[] {"name", "Johan", "extra", "Additional"}, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldAllowNodesWithoutIdHeader() throws Exception { // GIVEN DataFactory data = data( "name:string,level:int\n" + "Mattias,1\n" + "Johan,2\n" ); Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.STRING, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, null, new Object[] {"name", "Mattias", "level", 1}, labels() ); assertNextNode( nodes, null, new Object[] {"name", "Johan", "level", 2}, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldAllowSomeNodesToBeAnonymous() throws Exception { // GIVEN DataFactory data = data( ":ID,name:string,level:int\n" + "abc,Mattias,1\n" + ",Johan,2\n" ); // this node is anonymous Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.STRING, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, "abc", new Object[] {"name", "Mattias", "level", 1}, labels() ); assertNextNode( nodes, null, new Object[] {"name", "Johan", "level", 2}, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldAllowNodesToBeAnonymousEvenIfIdHeaderIsNamed() throws Exception { // GIVEN DataFactory data = data( "id:ID,name:string,level:int\n" + "abc,Mattias,1\n" + ",Johan,2\n" ); // this node is anonymous Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.STRING, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, "abc", new Object[] {"id", "abc", "name", "Mattias", "level", 1}, labels() ); assertNextNode( nodes, null, new Object[] {"name", "Johan", "level", 2}, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldNotHaveIdSetAsPropertyIfIdHeaderEntryIsNamedForActualIds() throws Exception { // GIVEN DataFactory data = data( "myId:ID,name:string,level:int\n" + "0,Mattias,1\n" + "1,Johan,2\n" ); // this node is anonymous Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, 0L, new Object[] {"name", "Mattias", "level", 1}, labels() ); assertNextNode( nodes, 1L, new Object[] {"name", "Johan", "level", 2}, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldIgnoreNodeEntriesMarkedIgnoreUsingHeader() throws Exception { // GIVEN Iterable<DataFactory> data = DataFactories.datas( CsvInputTest.data( ":ID,name:IGNORE,other:int,:LABEL\n" + "1,Mattias,10,Person\n" + "2,Johan,111,Person\n" + "3,Emil,12,Person" ) ); Input input = new CsvInput( data, defaultFormatNodeFileHeader(), datas(), defaultFormatNodeFileHeader(), IdType.INTEGER, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { assertNextNode( nodes, 1L, new Object[] {"other", 10}, labels( "Person" ) ); assertNextNode( nodes, 2L, new Object[] {"other", 111}, labels( "Person" ) ); assertNextNode( nodes, 3L, new Object[] {"other", 12}, labels( "Person" ) ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldNotIncludeEmptyArraysInEntities() throws Exception { // GIVEN Iterable<DataFactory> data = DataFactories.datas( CsvInputTest.data( ":ID,sprop:String[],lprop:long[]\n" + "1,,\n" + "2,a;b,10;20" ) ); Input input = new CsvInput( data, defaultFormatNodeFileHeader(), datas(), defaultFormatNodeFileHeader(), IdType.INTEGER, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN/THEN try ( InputIterator nodes = input.nodes().iterator() ) { assertNextNode( nodes, 1L, NO_PROPERTIES, labels() ); assertNextNode( nodes, 2L, properties( "sprop", new String[] {"a", "b"}, "lprop", new long[] {10, 20} ), labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldParseDatePropertyValues() throws Exception { // GIVEN DataFactory data = data( ":ID,name,date:Date\n" + "0,Mattias,2018-02-27\n" + "1,Johan,2018-03-01\n" ); Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, 0L, new Object[]{"name", "Mattias", "date", DateValue.date( 2018, 2, 27 )}, labels() ); assertNextNode( nodes, 1L, new Object[]{"name", "Johan", "date", DateValue.date( 2018, 3, 1 )}, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldParseDurationPropertyValues() throws Exception { // GIVEN DataFactory data = data( ":ID,name,duration:Duration\n" + "0,Mattias,P3MT13H37M\n" + "1,Johan,\"P-1YT4H20M\"\n" ); Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, 0L, new Object[]{"name", "Mattias", "duration", DurationValue.duration( 3, 0, 13 * 3600 + 37 * 60, 0 )}, labels() ); assertNextNode( nodes, 1L, new Object[]{"name", "Johan", "duration", DurationValue.duration( -12, 0, 4 * 3600 + 20 * 60, 0 )}, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldParseLocalTimePropertyValues() throws Exception { // GIVEN DataFactory data = data( ":ID,name,time:LocalTime\n" + "0,Mattias,13:37\n" + "1,Johan,\"16:20:01\"\n" ); Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, 0L, new Object[]{"name", "Mattias", "time", LocalTimeValue.localTime( 13, 37, 0, 0 )}, labels() ); assertNextNode( nodes, 1L, new Object[]{"name", "Johan", "time", LocalTimeValue.localTime( 16, 20, 1, 0 )}, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldParseLocalDateTimePropertyValues() throws Exception { // GIVEN DataFactory data = data( ":ID,name,time:LocalDateTime\n" + "0,Mattias,2018-02-27T13:37\n" + "1,Johan,\"2018-03-01T16:20:01\"\n" ); Iterable<DataFactory> dataIterable = dataIterable( data ); Input input = new CsvInput( dataIterable, defaultFormatNodeFileHeader(), datas(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, 0L, new Object[]{"name", "Mattias", "time", LocalDateTimeValue.localDateTime( 2018, 2, 27, 13, 37, 0, 0 )}, labels() ); assertNextNode( nodes, 1L, new Object[]{"name", "Johan", "time", LocalDateTimeValue.localDateTime( 2018, 3, 1, 16, 20, 1, 0 )}, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldIgnoreValuesAfterHeaderEntries() throws Exception { // GIVEN Iterable<DataFactory> data = dataIterable( data( "1,zergling,bubble,bobble\n" + "2,scv,pun,intended" ) ); Input input = new CsvInput( data, header( entry( null, Type.ID, extractors.long_() ), entry( "name", Type.PROPERTY, extractors.string() ) ), datas(), defaultFormatRelationshipFileHeader(), IdType.ACTUAL, config( COMMAS ), silentBadCollector( 4 ) ); // WHEN try ( InputIterator nodes = input.nodes().iterator() ) { // THEN assertNextNode( nodes, 1L, new Object[] { "name", "zergling" }, labels() ); assertNextNode( nodes, 2L, new Object[] { "name", "scv" }, labels() ); assertFalse( readNext( nodes ) ); } }
@Test public void shouldProvideNodesFromCsvInput() throws Exception { // GIVEN IdType idType = IdType.ACTUAL; Iterable<DataFactory> data = dataIterable( data( "123,Mattias Persson,HACKER" ) ); Input input = new CsvInput( data, header( entry( null, Type.ID, idType.extractor( extractors ) ), entry( "name", Type.PROPERTY, extractors.string() ), entry( "labels", Type.LABEL, extractors.string() ) ), datas(), defaultFormatRelationshipFileHeader(), idType, config( COMMAS ), silentBadCollector( 0 ) ); // WHEN/THEN try ( InputIterator nodes = input.nodes().iterator() ) { assertNextNode( nodes, 123L, properties( "name", "Mattias Persson" ), labels( "HACKER" ) ); assertFalse( chunk.next( visitor ) ); } }