public JsonTokenizer(final Reader input) { super(input); scanner = new JsonTokenizerImpl(input); this.initAttributes(); }
/** * Creates a new instance of the {@link TupleTokenizer}. Attaches the * <code>input</code> to a newly created JFlex scanner. */ public TupleTokenizer(final Reader input) { super(input); this._scanner = new TupleTokenizerImpl(input); this.initAttributes(); }
private static int [] zzUnpackRowMap() { int [] result = new int[118]; int offset = 0; offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); return result; }
public static Analyzer newTupleAnalyzer() { final AnyURIAnalyzer uriAnalyzer = new AnyURIAnalyzer(TEST_VERSION_CURRENT); uriAnalyzer.setUriNormalisation(URINormalisation.FULL); final TupleAnalyzer analyzer = new TupleAnalyzer(TEST_VERSION_CURRENT, new StandardAnalyzer(TEST_VERSION_CURRENT), uriAnalyzer); return analyzer; }
@Override protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) { final Tokenizer sink = new NumericTokenizer(reader, new DoubleNumericParser(), precisionStep); return new TokenStreamComponents(sink); }
@Override public void reset() { final MockSirenReader reader = (MockSirenReader) this.input; this.doc = reader.getDocument(); nodeIt = doc.iterator(); this.clearAttributes(); }
@Test public void testEmptyField() throws Exception { this.assertTokenizesTo(_t, "{ \"\" : \"b\" , }", new String[] { "", "b" }, new String[] { TOKEN_TYPES[LITERAL], TOKEN_TYPES[LITERAL] }, new String[] { JSON_FIELD, XSD_STRING }, new int[] { 1, 1 }, new IntsRef[] { node(0), node(0, 0) }); }
public void assertAnalyzesTo(final Analyzer a, final String input, final String[] expectedImages, final String[] expectedTypes) throws Exception { this.assertAnalyzesTo(a, input, expectedImages, expectedTypes, null); }
protected void assertTokenizesTo(final Tokenizer t, final String input, final String[] expectedImages, final String[] expectedTypes, final String[] expectedDatatypes) throws Exception { this.assertTokenizesTo(t, input, expectedImages, expectedTypes, expectedDatatypes, null, null); }
private static int [] zzUnpackRowMap() { int [] result = new int[32]; int offset = 0; offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); return result; }
private static int [] zzUnpackAction() { int [] result = new int[32]; int offset = 0; offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); return result; }
private static int [] zzUnpackTrans() { int [] result = new int[520]; int offset = 0; offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result); return result; }
private static int [] zzUnpackAction() { int [] result = new int[118]; int offset = 0; offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); return result; }
private static int [] zzUnpackAttribute() { int [] result = new int[118]; int offset = 0; offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); return result; }
private static int [] zzUnpackAttribute() { int [] result = new int[32]; int offset = 0; offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); return result; }
private static int [] zzUnpackTrans() { int [] result = new int[2640]; int offset = 0; offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result); return result; }
@Test(expected=IllegalStateException.class) public void testNoFieldJson1() throws Exception { this.assertTokenizesTo(_t, "{ : \"bad\" }", new String[] { "bad" }, new String[] { TOKEN_TYPES[LITERAL] }, new String[] { XSD_STRING }, new int[] { 1 }, new IntsRef[] { node(-1, 0) }); }
public void assertAnalyzesTo(final Analyzer a, final String input, final String[] expectedImages, final String[] expectedTypes, final int[] expectedPosIncrs) throws Exception { this.assertAnalyzesTo(a, input, expectedImages, expectedTypes, expectedPosIncrs, null, null); }
protected void assertTokenizesTo(final Tokenizer t, final String input, final String[] expectedImages, final String[] expectedTypes, final int[] expectedPosIncrs, final IntsRef[] expectedNode) throws Exception { this.assertTokenizesTo(t, input, expectedImages, expectedTypes, null, expectedPosIncrs, expectedNode); }
@Test(expected=IllegalStateException.class) public void testNoFieldJson4() throws Exception { this.assertTokenizesTo(_t, "{ : true }", new String[] { "bad" }, new String[] { TOKEN_TYPES[TRUE] }, new String[] { XSD_BOOLEAN }, new int[] { 1 }, new IntsRef[] { node(-1, 0) }); }