@Test public void testStringLiterals() { String[] stringLiterals = { "\"\"", "\"hi\"", "\"\\u00fe\"", "\"\\\"\"", "\"\\\\/\\b\\f\\n\\r\\t\"", }; for (String code : stringLiterals) { Segment segment = createSegment(code); JsonTokenMaker tm = new JsonTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertEquals("Invalid string: " + token, TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, token.getType()); } }
@Test public void testNoMultiLineComments() { String code = "// Hello world"; Segment segment = createSegment(code); JsonTokenMaker tm = new JsonTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); // Only need to check the first token Assert.assertFalse(token.isComment()); }
@Test public void testStringLiterals_errors() { String[] stringLiterals = { "\"foo \\x bar\"", "\"foo unterminated string", }; for (String code : stringLiterals) { Segment segment = createSegment(code); JsonTokenMaker tm = new JsonTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertEquals("Invalid error-string: " + token, TokenTypes.ERROR_STRING_DOUBLE, token.getType()); } }
@Test public void testNullLiterals() { String code = "null"; Segment segment = createSegment(code); JsonTokenMaker tm = new JsonTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.RESERVED_WORD, "null")); }
@Test public void testSeparators() { String code = "[ ] { }"; Segment segment = createSegment(code); JsonTokenMaker tm = new JsonTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] separators = code.split(" +"); for (int i = 0; i < separators.length; i++) { Assert.assertEquals(separators[i], token.getLexeme()); Assert.assertEquals(TokenTypes.SEPARATOR, token.getType()); // Just one extra test here Assert.assertTrue(token.isSingleChar(TokenTypes.SEPARATOR, separators[i].charAt(0))); if (i < separators.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue("Not a single space: " + token, token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
@Test public void testIntegerLiterals() { String code = "1 42 0 -1 -42"; Segment segment = createSegment(code); JsonTokenMaker tm = new JsonTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals("Unexpected number for token " + i, keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.LITERAL_NUMBER_DECIMAL_INT, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
@Test public void testBooleanLiterals() { String code = "true false"; Segment segment = createSegment(code); JsonTokenMaker tm = new JsonTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.LITERAL_BOOLEAN, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0);