public String head() { return " " + precToken.fixedText() + " "; } public String sep() { return ", "; }
public static final boolean isSeparator(JavaTokenId id) { return "separator".equals(id.primaryCategory()); //NOI18N }
@NonNull public final TokenHierarchy<?> getTokenHierarchy() throws IOException { if (this.tokens == null) { final CharBuffer charBuffer = getCharContent(true); this.tokens = TokenHierarchy.create(charBuffer, false, JavaTokenId.language(), null, null); //TODO: .createSnapshot(); } return this.tokens; }
public static TokenSequence<JavadocTokenId> findJavadocTokenSequence(CompilationInfo javac, int offset) { TokenSequence<JavaTokenId> ts = SourceUtils.getJavaTokenSequence(javac.getTokenHierarchy(), offset); if (ts == null || !movedToJavadocToken(ts, offset)) { return null; } TokenSequence<JavadocTokenId> jdts = ts.embedded(JavadocTokenId.language()); if (jdts == null) { return null; } jdts.move(offset); return jdts; }
static int moveFwdToToken(TokenSequence<JavaTokenId> tokenSequence, final int pos, JavaTokenId id) { tokenSequence.move(pos); tokenSequence.moveNext(); // Assumes the pos is located within input bounds while (!id.equals(tokenSequence.token().id())) { if (!tokenSequence.moveNext()) return -1; } return tokenSequence.offset(); }
private void mapComments(BlockTree block, String inputText, WorkingCopy copy, CommentHandler comments, SourcePositions positions) { TokenSequence<JavaTokenId> seq = TokenHierarchy.create(inputText, JavaTokenId.language()).tokenSequence(JavaTokenId.language()); AssignComments ti = new AssignComments(copy, block, seq, positions); ti.scan(block, null);
private static TokenSequence<JavadocTokenId> getJavadocTS(CompilationInfo javac, int start) { TokenSequence<JavadocTokenId> javadoc = null; TokenSequence<JavaTokenId> ts = SourceUtils.getJavaTokenSequence(javac.getTokenHierarchy(), start); if (ts.moveNext() && ts.token().id() == JavaTokenId.JAVADOC_COMMENT) { javadoc = ts.embedded(JavadocTokenId.language()); } return javadoc; }
static int moveBackToToken(TokenSequence<JavaTokenId> tokenSequence, final int pos, JavaTokenId id) { tokenSequence.move(pos); tokenSequence.moveNext(); // Assumes the pos is located within input bounds while (!id.equals(tokenSequence.token().id())) { if (!tokenSequence.movePrevious()) return -1; } return tokenSequence.offset(); }
@Override public void run() { if (TokenList.this.cancel.get()) { return ; } topLevel = TokenHierarchy.get(doc).tokenSequence(); topLevelIsJava = topLevel.language() == JavaTokenId.language(); if (topLevelIsJava) { ts = topLevel; ts.moveStart(); ts.moveNext(); //XXX: what about empty document } } });
public static boolean isJavadocContext(TokenHierarchy hierarchy, int offset) { TokenSequence<JavaTokenId> ts = SourceUtils.getJavaTokenSequence(hierarchy, offset); if (!movedToJavadocToken(ts, offset)) { return false; } TokenSequence<JavadocTokenId> jdts = ts.embedded(JavadocTokenId.language()); if (jdts == null) { return false; } else if (jdts.isEmpty()) { return isEmptyJavadoc(ts.token(), offset - ts.offset()); } jdts.move(offset); if (!jdts.moveNext() && !jdts.movePrevious()) { return false; } // this checks /** and */ headers return isInsideToken(jdts, offset) && !isInsideIndent(jdts.token(), offset - jdts.offset()); }
private static void lazyInit() { if (inited.compareAndSet(false, true)) { preferences = MimeLookup.getLookup(JavaTokenId.language().mimeType()).lookup(Preferences.class); preferences.addPreferenceChangeListener(WeakListeners.create(PreferenceChangeListener.class, preferencesTracker, preferences)); preferencesTracker.preferenceChange(null); } }
private Pretty(CompilationInfo info, TreePath path, CodeStyle cs, int startOffset, int endOffset, boolean templateEdit) { this(info.getText(), info.getTokenHierarchy().tokenSequence(JavaTokenId.language()), path, info.getTrees().getSourcePositions(), cs, startOffset, endOffset, cs.getRightMargin()); this.templateEdit = templateEdit; }
TokenSequence<JavaTokenId> findLastNonWhitespaceToken(Env env, int startPos, int endPos) { TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language()); ts.move(endPos); ts = previousNonWhitespaceToken(ts); if (ts == null || ts.offset() < startPos) { return null; } return ts; }
/** * @since 0.21 */ public static TokenSequence<JavaTokenId> getJavaTokenSequence(final TokenHierarchy hierarchy, final int offset) { if (hierarchy != null) { TokenSequence<?> ts = hierarchy.tokenSequence(); while(ts != null && (offset == 0 || ts.moveNext())) { ts.move(offset); if (ts.language() == JavaTokenId.language()) { return (TokenSequence<JavaTokenId>)ts; } if (!ts.moveNext() && !ts.movePrevious()) { return null; } ts = ts.embedded(); } } return null; }
TokenSequence<JavaTokenId> findFirstNonWhitespaceToken(Env env, int startPos, int endPos) { TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language()); ts.move(startPos); ts = nextNonWhitespaceToken(ts); if (ts == null || ts.offset() >= endPos) { return null; } return ts; }
/**Returns tokens for a given tree. Uses specified {@link SourcePositions}. */ public TokenSequence<JavaTokenId> tokensFor(Tree tree, SourcePositions sourcePositions) { int start = (int)sourcePositions.getStartPosition(info.getCompilationUnit(), tree); int end = (int)sourcePositions.getEndPosition(info.getCompilationUnit(), tree); return info.getTokenHierarchy().tokenSequence(JavaTokenId.language()).subSequence(start, end); }
private static Token<JavaTokenId> findIdentifierSpanImpl(CompilationInfo info, IdentifierTree tree, CompilationUnitTree cu, SourcePositions positions) { int start = (int)positions.getStartPosition(cu, tree); int endPosition = (int)positions.getEndPosition(cu, tree); if (start == (-1) || endPosition == (-1)) return null; TokenHierarchy<?> th = info.getTokenHierarchy(); TokenSequence<JavaTokenId> ts = th.tokenSequence(JavaTokenId.language()); if (ts.move(start) == Integer.MAX_VALUE) { return null; } if (ts.moveNext()) { if (ts.offset() >= start) { Token<JavaTokenId> t = ts.token(); return t; } } return null; }
@Override protected LanguageEmbedding<?> embedding( Token<JspTokenId> token, LanguagePath languagePath, InputAttributes inputAttributes) { switch(token.id()) { case TEXT: return LanguageEmbedding.create(HTMLTokenId.language(), 0, 0, true); case EL: //lexer infrastructure workaround - need to adjust skiplenghts in case of short token int startSkipLength = token.length() > 2 ? 2 : token.length(); int endSkipLength = token.length() > 2 ? 1 : 0; return LanguageEmbedding.create(ELTokenId.language(), startSkipLength, endSkipLength); case SCRIPTLET: return LanguageEmbedding.create(JavaTokenId.language(), 0, 0, false); default: return null; } }
public void checkInitialFold() { try { TokenHierarchy<?> th = info.getTokenHierarchy(); TokenSequence<JavaTokenId> ts = th.tokenSequence(JavaTokenId.language()); while (ts.moveNext()) { if (ts.offset() >= initialCommentStopPos) break; Token<JavaTokenId> token = ts.token(); if (token.id() == JavaTokenId.BLOCK_COMMENT || token.id() == JavaTokenId.JAVADOC_COMMENT) { int startOffset = ts.offset(); addFold(creator.createInitialCommentFold(startOffset, startOffset + token.length()), startOffset); break; } } } catch (ConcurrentModificationException e) { //from TokenSequence, document probably changed, stop stopped = true; } }