@NonNull public final TokenHierarchy<?> getTokenHierarchy() throws IOException { if (this.tokens == null) { final CharBuffer charBuffer = getCharContent(true); this.tokens = TokenHierarchy.create(charBuffer, false, JavaTokenId.language(), null, null); //TODO: .createSnapshot(); } return this.tokens; }
private static void lazyInit() { if (inited.compareAndSet(false, true)) { preferences = MimeLookup.getLookup(JavaTokenId.language().mimeType()).lookup(Preferences.class); preferences.addPreferenceChangeListener(WeakListeners.create(PreferenceChangeListener.class, preferencesTracker, preferences)); preferencesTracker.preferenceChange(null); } }
@Override public void run() { if (TokenList.this.cancel.get()) { return ; } topLevel = TokenHierarchy.get(doc).tokenSequence(); topLevelIsJava = topLevel.language() == JavaTokenId.language(); if (topLevelIsJava) { ts = topLevel; ts.moveStart(); ts.moveNext(); //XXX: what about empty document } } });
public static String reformat(String text, CodeStyle style, int rightMargin) { StringBuilder sb = new StringBuilder(text); try { ClassPath empty = ClassPathSupport.createClassPath(new URL[0]); ClasspathInfo cpInfo = ClasspathInfo.create(JavaPlatformManager.getDefault().getDefaultPlatform().getBootstrapLibraries(), empty, empty); JavacTaskImpl javacTask = JavacParser.createJavacTask(cpInfo, null, null, null, null, null, null, null); com.sun.tools.javac.util.Context ctx = javacTask.getContext(); JavaCompiler.instance(ctx).genEndPos = true; CompilationUnitTree tree = javacTask.parse(FileObjects.memoryFileObject("","", text)).iterator().next(); //NOI18N SourcePositions sp = JavacTrees.instance(ctx).getSourcePositions(); TokenSequence<JavaTokenId> tokens = TokenHierarchy.create(text, JavaTokenId.language()).tokenSequence(JavaTokenId.language()); for (Diff diff : Pretty.reformat(text, tokens, new TreePath(tree), sp, style, rightMargin)) { int start = diff.getStartOffset(); int end = diff.getEndOffset(); sb.delete(start, end); String t = diff.getText(); if (t != null && t.length() > 0) { sb.insert(start, t); } } } catch (IOException ioe) { } return sb.toString(); }
private Pretty(CompilationInfo info, TreePath path, CodeStyle cs, int startOffset, int endOffset, boolean templateEdit) { this(info.getText(), info.getTokenHierarchy().tokenSequence(JavaTokenId.language()), path, info.getTrees().getSourcePositions(), cs, startOffset, endOffset, cs.getRightMargin()); this.templateEdit = templateEdit; }
TokenSequence<JavaTokenId> findLastNonWhitespaceToken(Env env, int startPos, int endPos) { TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language()); ts.move(endPos); ts = previousNonWhitespaceToken(ts); if (ts == null || ts.offset() < startPos) { return null; } return ts; }
TokenSequence<JavaTokenId> findFirstNonWhitespaceToken(Env env, int startPos, int endPos) { TokenSequence<JavaTokenId> ts = env.getController().getTokenHierarchy().tokenSequence(JavaTokenId.language()); ts.move(startPos); ts = nextNonWhitespaceToken(ts); if (ts == null || ts.offset() >= endPos) { return null; } return ts; }
private static Token<JavaTokenId> findTokenWithText(CompilationInfo info, String text, int start, int end) { TokenHierarchy<?> th = info.getTokenHierarchy(); TokenSequence<JavaTokenId> ts = th.tokenSequence(JavaTokenId.language()).subSequence(start, end); while (ts.moveNext()) { Token<JavaTokenId> t = ts.token(); if (t.id() == JavaTokenId.IDENTIFIER) { boolean nameMatches; if (!(nameMatches = text.equals(info.getTreeUtilities().decodeIdentifier(t.text()).toString()))) { ExpressionTree expr = info.getTreeUtilities().parseExpression(t.text().toString(), new SourcePositions[1]); nameMatches = expr.getKind() == Kind.IDENTIFIER && text.contentEquals(((IdentifierTree) expr).getName()); } if (nameMatches) { return t; } } } return null; }
/**Returns tokens for a given tree. Uses specified {@link SourcePositions}. */ public TokenSequence<JavaTokenId> tokensFor(Tree tree, SourcePositions sourcePositions) { int start = (int)sourcePositions.getStartPosition(info.getCompilationUnit(), tree); int end = (int)sourcePositions.getEndPosition(info.getCompilationUnit(), tree); return info.getTokenHierarchy().tokenSequence(JavaTokenId.language()).subSequence(start, end); }
/** * Note: the constructor is only used when creating new compilation units; a special flag will be set up, so an initial comment is not skipped. */ public DiffContext(CompilationInfo copy, CompilationUnitTree cut, String code, PositionConverter positionConverter, FileObject file, Set<Tree> syntheticTrees, CompilationUnitTree mainUnit, String mainCode) { this.tokenSequence = TokenHierarchy.create(code, JavaTokenId.language()).tokenSequence(JavaTokenId.language()); this.origText = code; this.style = getCodeStyle(copy); this.context = JavaSourceAccessor.getINSTANCE().getJavacTask(copy).getContext(); this.origUnit = (JCCompilationUnit) cut; this.trees = copy.getTrees(); this.doc = null; this.positionConverter = positionConverter; this.file = file; this.syntheticTrees = syntheticTrees; this.mainUnit = (JCCompilationUnit) mainUnit; this.mainCode = mainCode; this.textLength = copy.getSnapshot() == null ? Integer.MAX_VALUE : copy.getSnapshot().getOriginalOffset(copy.getSnapshot().getText().length()); this.blockSequences = new BlockSequences(this.tokenSequence, doc, textLength); this.forceInitialComment = true; }
private static Token<JavaTokenId> findIdentifierSpanImpl(CompilationInfo info, MemberSelectTree tree, CompilationUnitTree cu, SourcePositions positions) { int start = (int)positions.getStartPosition(cu, tree); int endPosition = (int)positions.getEndPosition(cu, tree); if (start == (-1) || endPosition == (-1)) return null; String member = tree.getIdentifier().toString(); TokenHierarchy<?> th = info.getTokenHierarchy(); TokenSequence<JavaTokenId> ts = th.tokenSequence(JavaTokenId.language()); if (ts.move(endPosition) == Integer.MAX_VALUE) { return null; } if (ts.moveNext()) { while (ts.offset() >= start) { Token<JavaTokenId> t = ts.token(); if (t.id() == JavaTokenId.IDENTIFIER && member.equals(info.getTreeUtilities().decodeIdentifier(t.text()).toString())) { return t; } if (!ts.movePrevious()) break; } } return null; }
private static Token<JavaTokenId> findIdentifierSpanImpl(CompilationInfo info, MemberReferenceTree tree, CompilationUnitTree cu, SourcePositions positions) { int start = (int)positions.getStartPosition(cu, tree); int endPosition = (int)positions.getEndPosition(cu, tree); if (start == (-1) || endPosition == (-1)) return null; String member = tree.getName().toString(); TokenHierarchy<?> th = info.getTokenHierarchy(); TokenSequence<JavaTokenId> ts = th.tokenSequence(JavaTokenId.language()); if (ts.move(endPosition) == Integer.MAX_VALUE) { return null; } if (ts.moveNext()) { while (ts.offset() >= start) { Token<JavaTokenId> t = ts.token(); if (t.id() == JavaTokenId.IDENTIFIER && member.equals(info.getTreeUtilities().decodeIdentifier(t.text()).toString())) { return t; } if (!ts.movePrevious()) break; } } return null; }
/** * @since 0.21 */ public static TokenSequence<JavaTokenId> getJavaTokenSequence(final TokenHierarchy hierarchy, final int offset) { if (hierarchy != null) { TokenSequence<?> ts = hierarchy.tokenSequence(); while(ts != null && (offset == 0 || ts.moveNext())) { ts.move(offset); if (ts.language() == JavaTokenId.language()) { return (TokenSequence<JavaTokenId>)ts; } if (!ts.moveNext() && !ts.movePrevious()) { return null; } ts = ts.embedded(); } } return null; }
private static Token<JavaTokenId> findIdentifierSpanImpl(CompilationInfo info, IdentifierTree tree, CompilationUnitTree cu, SourcePositions positions) { int start = (int)positions.getStartPosition(cu, tree); int endPosition = (int)positions.getEndPosition(cu, tree); if (start == (-1) || endPosition == (-1)) return null; TokenHierarchy<?> th = info.getTokenHierarchy(); TokenSequence<JavaTokenId> ts = th.tokenSequence(JavaTokenId.language()); if (ts.move(start) == Integer.MAX_VALUE) { return null; } if (ts.moveNext()) { if (ts.offset() >= start) { Token<JavaTokenId> t = ts.token(); return t; } } return null; }
@Override protected LanguageEmbedding<?> embedding( Token<JspTokenId> token, LanguagePath languagePath, InputAttributes inputAttributes) { switch(token.id()) { case TEXT: return LanguageEmbedding.create(HTMLTokenId.language(), 0, 0, true); case EL: //lexer infrastructure workaround - need to adjust skiplenghts in case of short token int startSkipLength = token.length() > 2 ? 2 : token.length(); int endSkipLength = token.length() > 2 ? 1 : 0; return LanguageEmbedding.create(ELTokenId.language(), startSkipLength, endSkipLength); case SCRIPTLET: return LanguageEmbedding.create(JavaTokenId.language(), 0, 0, false); default: return null; } }
public void checkInitialFold() { try { TokenHierarchy<?> th = info.getTokenHierarchy(); TokenSequence<JavaTokenId> ts = th.tokenSequence(JavaTokenId.language()); while (ts.moveNext()) { if (ts.offset() >= initialCommentStopPos) break; Token<JavaTokenId> token = ts.token(); if (token.id() == JavaTokenId.BLOCK_COMMENT || token.id() == JavaTokenId.JAVADOC_COMMENT) { int startOffset = ts.offset(); addFold(creator.createInitialCommentFold(startOffset, startOffset + token.length()), startOffset); break; } } } catch (ConcurrentModificationException e) { //from TokenSequence, document probably changed, stop stopped = true; } }
public DiffContext(CompilationInfo copy, Set<Tree> syntheticTrees) { this.tokenSequence = copy.getTokenHierarchy().tokenSequence(JavaTokenId.language()); this.mainCode = this.origText = copy.getText(); this.style = getCodeStyle(copy); this.context = JavaSourceAccessor.getINSTANCE().getJavacTask(copy).getContext(); this.mainUnit = this.origUnit = (JCCompilationUnit) copy.getCompilationUnit(); this.trees = copy.getTrees(); this.doc = copy.getSnapshot().getSource().getDocument(false); //TODO: true or false? this.positionConverter = copy.getPositionConverter(); this.file = copy.getFileObject(); this.syntheticTrees = syntheticTrees; this.textLength = copy.getSnapshot() == null ? Integer.MAX_VALUE : copy.getSnapshot().getOriginalOffset(copy.getSnapshot().getText().length()); this.blockSequences = new BlockSequences(this.tokenSequence, doc, textLength); this.forceInitialComment = false; }
return null; TokenSequence<JavaTokenId> tokenSequence = info.getTokenHierarchy().tokenSequence(JavaTokenId.language()); tokenSequence.move(pos);
private void handleJavadoc(Tree t) throws BadLocationException, ConcurrentModificationException { int start = (int) sp.getStartPosition(cu, t); if (start == (-1)) return ; if (start < initialCommentStopPos) initialCommentStopPos = start; TokenHierarchy<?> th = info.getTokenHierarchy(); TokenSequence<JavaTokenId> ts = th.tokenSequence(JavaTokenId.language()); if (ts.move(start) == Integer.MAX_VALUE) { return;//nothing } while (ts.movePrevious()) { Token<JavaTokenId> token = ts.token(); if (token.id() == JavaTokenId.JAVADOC_COMMENT) { int startOffset = ts.offset(); addFold(creator.createJavadocFold(startOffset, startOffset + token.length()), startOffset); if (startOffset < initialCommentStopPos) initialCommentStopPos = startOffset; } if ( token.id() != JavaTokenId.WHITESPACE && token.id() != JavaTokenId.BLOCK_COMMENT && token.id() != JavaTokenId.LINE_COMMENT) break; } }
return null; TokenSequence<JavaTokenId> tokenSequence = info.getTokenHierarchy().tokenSequence(JavaTokenId.language());