@Override public List<String> tokenize(String source) { try { char[] charArray = source.toCharArray(); scanner.setSource(source.toCharArray()); List<String> tokens = new ArrayList<>(); int token; while ((token = scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) { int tokenStart = scanner.getCurrentTokenStartPosition(); int tokenEnd = scanner.getCurrentTokenEndPosition(); if (token != ITerminalSymbols.TokenNameWHITESPACE) { tokens.add(new String(charArray, tokenStart, tokenEnd - tokenStart + 1)); } } return tokens; } catch (InvalidInputException e) { throw new RuntimeException(e); } }
/** * Method to check with methods share common attributes, according to * CK definition. * @author Mariana Azevedo * @since 13/07/2014 * @param methods */ private void checkMethodsWithSharedAttributes(IMethod[] methods){ IScanner scanner = null; for (IMethod method : methods) { String methodName = method.getElementName(); try { scanner = ToolFactory.createScanner(false, false, false, false); scanner.setSource(method.getSource().toCharArray()); while(true){ int charactere = scanner.getNextToken(); if (charactere == ITerminalSymbols.TokenNameEOF) break; if (charactere == ITerminalSymbols.TokenNameIdentifier) { addMethods(new String(scanner.getCurrentTokenSource()), methodName); } } } catch (JavaModelException exception1) { logger.error(exception1); } catch (InvalidInputException exception2) { logger.error(exception2); } } }
/** * Removes comments and whitespace * * @param reference * the type reference * @return the reference only consisting of dots and java identifier * characters */ public static String normalizeReference(String reference) { IScanner scanner = ToolFactory.createScanner(false, false, false, false); scanner.setSource(reference.toCharArray()); StringBuffer sb = new StringBuffer(); try { int tokenType = scanner.getNextToken(); while (tokenType != ITerminalSymbols.TokenNameEOF) { sb.append(scanner.getRawTokenSource()); tokenType = scanner.getNextToken(); } } catch (InvalidInputException e) { Assert.isTrue(false, reference); } reference = sb.toString(); return reference; }
private static boolean isJustWhitespaceOrComment(int start, int end, IBuffer buffer) { if (start == end) return true; Assert.isTrue(start <= end); String trimmedText = buffer.getText(start, end - start).trim(); if (0 == trimmedText.length()) { return true; } else { IScanner scanner = ToolFactory.createScanner(false, false, false, null); scanner.setSource(trimmedText.toCharArray()); try { return scanner.getNextToken() == ITerminalSymbols.TokenNameEOF; } catch (InvalidInputException e) { return false; } } }
private void doScan() { try { int token = fScanner.getNextToken(); while (token != ITerminalSymbols.TokenNameEOF) { switch (token) { case ITerminalSymbols.TokenNameStringLiteral: case ITerminalSymbols.TokenNameCOMMENT_JAVADOC: case ITerminalSymbols.TokenNameCOMMENT_LINE: case ITerminalSymbols.TokenNameCOMMENT_BLOCK: parseCurrentToken(); } token = fScanner.getNextToken(); } } catch (InvalidInputException e) { // ignore } }
/** * Removes comments and whitespace * * @param reference the type reference * @return the reference only consisting of dots and java identifier characters */ public static String normalizeReference(String reference) { IScanner scanner = ToolFactory.createScanner(false, false, false, false); scanner.setSource(reference.toCharArray()); StringBuffer sb = new StringBuffer(); try { int tokenType = scanner.getNextToken(); while (tokenType != ITerminalSymbols.TokenNameEOF) { sb.append(scanner.getRawTokenSource()); tokenType = scanner.getNextToken(); } } catch (InvalidInputException e) { Assert.isTrue(false, reference); } reference = sb.toString(); return reference; }
private int getSurroundingComment(IScanner scanner) { try { int start = fLocation.getOffset(); int end = start + fLocation.getLength(); int token = scanner.getNextToken(); while (token != ITerminalSymbols.TokenNameEOF) { if (TokenScanner.isComment(token)) { int currStart = scanner.getCurrentTokenStartPosition(); int currEnd = scanner.getCurrentTokenEndPosition() + 1; if (currStart <= start && end <= currEnd) { return token; } } token = scanner.getNextToken(); } } catch (InvalidInputException e) { // ignore } return ITerminalSymbols.TokenNameEOF; }
/** * This method is used to retrieve the start position of the block. * @return int the dimension found, -1 if none */ protected int retrieveStartBlockPosition(int start, int end) { this.scanner.resetTo(start, end); try { int token; while ((token = this.scanner.getNextToken()) != TerminalTokens.TokenNameEOF) { switch(token) { case TerminalTokens.TokenNameLBRACE://110 return this.scanner.startPosition; } } } catch(InvalidInputException e) { // ignore } return -1; }
@Override public List<String> tokenListFromCode(final char[] code) { final List<String> tokens = Lists.newArrayList(); tokens.add(SENTENCE_START); final PublicScanner scanner = prepareScanner(code); do { try { final int token = scanner.getNextToken(); if (token == ITerminalSymbols.TokenNameEOF) { break; } tokens.addAll(getConvertedToken(scanner, token)); } catch (final InvalidInputException e) { LOGGER.warning(ExceptionUtils.getFullStackTrace(e)); } } while (!scanner.atEnd()); tokens.add(SENTENCE_END); return tokens; }
@Override public List<FullToken> getTokenListFromCode(final char[] code) { final List<FullToken> tokens = Lists.newArrayList(); tokens.add(new FullToken(SENTENCE_START, SENTENCE_START)); final PublicScanner scanner = prepareScanner(code); do { try { final int token = scanner.getNextToken(); if (token == ITerminalSymbols.TokenNameEOF) { break; } for (final String cToken : getConvertedToken(scanner, token)) { tokens.add(new FullToken(cToken, "")); } } catch (final InvalidInputException e) { LOGGER.warning(ExceptionUtils.getFullStackTrace(e)); } } while (!scanner.atEnd()); tokens.add(new FullToken(SENTENCE_END, SENTENCE_END)); return tokens; }
private final void consumeDigits(int radix, boolean expectingDigitFirst) throws InvalidInputException { final int USING_UNDERSCORE = 1; final int INVALID_POSITION = 2; switch(consumeDigits0(radix, USING_UNDERSCORE, INVALID_POSITION, expectingDigitFirst)) { case USING_UNDERSCORE : if (this.sourceLevel < ClassFileConstants.JDK1_7) { throw new InvalidInputException(UNDERSCORES_IN_LITERALS_NOT_BELOW_17); } break; case INVALID_POSITION : if (this.sourceLevel < ClassFileConstants.JDK1_7) { throw new InvalidInputException(UNDERSCORES_IN_LITERALS_NOT_BELOW_17); } throw new InvalidInputException(INVALID_UNDERSCORE); } }
/** * This method is used to retrieve the position after the right parenthesis. * @return int the position found */ protected int retrieveEndOfRightParenthesisPosition(int start, int end) { this.scanner.resetTo(start, end); try { int token; while ((token = this.scanner.getNextToken()) != TerminalTokens.TokenNameEOF) { switch(token) { case TerminalTokens.TokenNameRPAREN: return this.scanner.currentPosition; } } } catch(InvalidInputException e) { // ignore } return -1; }
private void doScan() { try{ int token = fScanner.getNextToken(); while (token != ITerminalSymbols.TokenNameEOF) { switch (token) { case ITerminalSymbols.TokenNameStringLiteral : case ITerminalSymbols.TokenNameCOMMENT_JAVADOC : case ITerminalSymbols.TokenNameCOMMENT_LINE : case ITerminalSymbols.TokenNameCOMMENT_BLOCK : parseCurrentToken(); } token = fScanner.getNextToken(); } } catch (InvalidInputException e){ //ignore } }
private static boolean isJustWhitespaceOrComment(int start, int end, IBuffer buffer) { if (start == end) return true; Assert.isTrue(start <= end); String trimmedText= buffer.getText(start, end - start).trim(); if (0 == trimmedText.length()) { return true; } else { IScanner scanner= ToolFactory.createScanner(false, false, false, null); scanner.setSource(trimmedText.toCharArray()); try { return scanner.getNextToken() == ITerminalSymbols.TokenNameEOF; } catch (InvalidInputException e) { return false; } } }
protected Object createFieldReference(Object receiver) throws InvalidInputException { try { MemberRef fieldRef = this.ast.newMemberRef(); SimpleName fieldName = new SimpleName(this.ast); fieldName.internalSetIdentifier(new String(this.identifierStack[0])); fieldRef.setName(fieldName); int start = (int) (this.identifierPositionStack[0] >>> 32); int end = (int) this.identifierPositionStack[0]; fieldName.setSourceRange(start, end - start + 1); if (receiver == null) { start = this.memberStart; fieldRef.setSourceRange(start, end - start + 1); } else { Name typeRef = (Name) receiver; fieldRef.setQualifier(typeRef); start = typeRef.getStartPosition(); end = fieldName.getStartPosition()+fieldName.getLength()-1; fieldRef.setSourceRange(start, end-start+1); } return fieldRef; } catch (ClassCastException ex) { throw new InvalidInputException(); } }
protected Object syntaxRecoverQualifiedName(int primitiveToken) throws InvalidInputException { if (this.cursorLocation == ((int)this.identifierPositionStack[this.identifierPtr])) { // special case of completion just before the dot. return createTypeReference(primitiveToken); } int idLength = this.identifierLengthStack[this.identifierLengthPtr]; char[][] tokens = new char[idLength][]; int startPtr = this.identifierPtr-idLength+1; System.arraycopy(this.identifierStack, startPtr, tokens, 0, idLength); long[] positions = new long[idLength+1]; System.arraycopy(this.identifierPositionStack, startPtr, positions, 0, idLength); positions[idLength] = (((long)this.tokenPreviousPosition)<<32) + this.tokenPreviousPosition; this.completionNode = new CompletionOnJavadocQualifiedTypeReference(tokens, CharOperation.NO_CHAR, positions, this.tagSourceStart, this.tagSourceEnd); if (CompletionEngine.DEBUG) { System.out.println(" completion partial qualified type="+this.completionNode); //$NON-NLS-1$ } return this.completionNode; }
/** * This method is used to retrieve position before the next right brace or semi-colon. * @return int the position found. */ protected int retrieveRightBrace(int start, int end) { this.scanner.resetTo(start, end); try { int token; while ((token = this.scanner.getNextToken()) != TerminalTokens.TokenNameEOF) { switch(token) { case TerminalTokens.TokenNameRBRACE : return this.scanner.currentPosition - 1; } } } catch(InvalidInputException e) { // ignore } return -1; }
protected Object createFieldReference(Object receiver) throws InvalidInputException { int refStart = (int) (this.identifierPositionStack[0] >>> 32); int refEnd = (int) this.identifierPositionStack[0]; boolean inCompletion = (refStart <= (this.cursorLocation+1) && this.cursorLocation <= refEnd) // completion cursor is between first and last stacked identifiers || ((refStart == (refEnd+1) && refEnd == this.cursorLocation)) // or it's a completion on empty token || (this.memberStart == this.cursorLocation); // or it's a completion just after the member separator with an identifier after the cursor if (inCompletion) { JavadocFieldReference fieldRef = (JavadocFieldReference) super.createFieldReference(receiver); char[] name = this.sourceParser.compilationUnit.getMainTypeName(); TypeDeclaration typeDecl = getParsedTypeDeclaration(); if (typeDecl != null) { name = typeDecl.name; } this.completionNode = new CompletionOnJavadocFieldReference(fieldRef, this.memberStart, name); if (CompletionEngine.DEBUG) { System.out.println(" completion field="+this.completionNode); //$NON-NLS-1$ } return this.completionNode; } return super.createFieldReference(receiver); }
public void printNextToken(int[] expectedTokenTypes, boolean considerSpaceIfAny){ printComment(CodeFormatter.K_UNKNOWN, NO_TRAILING_COMMENT); try { this.currentToken = this.scanner.getNextToken(); if (Arrays.binarySearch(expectedTokenTypes, this.currentToken) < 0) { StringBuffer expectations = new StringBuffer(5); for (int i = 0; i < expectedTokenTypes.length; i++){ if (i > 0) { expectations.append(','); } expectations.append(expectedTokenTypes[i]); } throw new AbortFormatting("unexpected token type, expecting:["+expectations.toString()+"], actual:"+this.currentToken);//$NON-NLS-1$//$NON-NLS-2$ } print(this.scanner.currentPosition - this.scanner.startPosition, considerSpaceIfAny); } catch (InvalidInputException e) { throw new AbortFormatting(e); } }
private boolean isNextToken(int tokenName) { this.localScanner.resetTo(this.scribe.scanner.currentPosition, this.scribe.scannerEndPosition - 1); try { int token = this.localScanner.getNextToken(); loop: while(true) { switch(token) { case TerminalTokens.TokenNameCOMMENT_BLOCK : case TerminalTokens.TokenNameCOMMENT_JAVADOC : case TerminalTokens.TokenNameCOMMENT_LINE : token = this.localScanner.getNextToken(); continue loop; default: break loop; } } return token == tokenName; } catch(InvalidInputException e) { // ignore } return false; }
private boolean isPartOfMultipleLocalDeclaration() { this.localScanner.resetTo(this.scribe.scanner.currentPosition, this.scribe.scannerEndPosition - 1); try { int token; while ((token = this.localScanner.getNextToken()) != TerminalTokens.TokenNameEOF) { switch(token) { case TerminalTokens.TokenNameCOMMA ://90 return true; case TerminalTokens.TokenNameCOMMENT_BLOCK : case TerminalTokens.TokenNameCOMMENT_JAVADOC : case TerminalTokens.TokenNameCOMMENT_LINE : break; default: return false; } } } catch(InvalidInputException e) { // ignore } return false; }
private boolean isClosingGenericToken() { this.localScanner.resetTo(this.scribe.scanner.currentPosition, this.scribe.scannerEndPosition - 1); try { int token = this.localScanner.getNextToken(); loop: while(true) { switch(token) { case TerminalTokens.TokenNameCOMMENT_BLOCK : case TerminalTokens.TokenNameCOMMENT_JAVADOC : case TerminalTokens.TokenNameCOMMENT_LINE : token = this.localScanner.getNextToken(); continue loop; default: break loop; } } switch(token) { case TerminalTokens.TokenNameGREATER : case TerminalTokens.TokenNameRIGHT_SHIFT : case TerminalTokens.TokenNameUNSIGNED_RIGHT_SHIFT : return true; } } catch(InvalidInputException e) { // ignore } return false; }
/** * This method is used to retrieve the array dimension declared after the * name of a local or a field declaration. * For example: * int i, j[] = null, k[][] = {{}}; * It should return 0 for i, 1 for j and 2 for k. * @return int the dimension found */ protected int retrieveExtraDimension(int start, int end) { this.scanner.resetTo(start, end); int dimensions = 0; try { int token; while ((token = this.scanner.getNextToken()) != TerminalTokens.TokenNameEOF) { switch(token) { case TerminalTokens.TokenNameLBRACKET: case TerminalTokens.TokenNameCOMMENT_BLOCK: case TerminalTokens.TokenNameCOMMENT_JAVADOC: case TerminalTokens.TokenNameCOMMENT_LINE: break; case TerminalTokens.TokenNameRBRACKET://166 dimensions++; break; default: return dimensions; } } } catch(InvalidInputException e) { // ignore } return dimensions; }
/** * Sets the string value of this literal node to the given string literal * token. The token is the sequence of characters that would appear in the * source program, including enclosing double quotes and embedded escapes. * For example, * <ul> * <li><code>""</code> <code>setLiteral("\"\"")</code></li> * <li><code>"hello world"</code> <code>setLiteral("\"hello world\"")</code></li> * <li><code>"boo\nhoo"</code> <code>setLiteral("\"boo\\nhoo\"")</code></li> * </ul> * * @param token the string literal token, including enclosing double * quotes and embedded escapes * @exception IllegalArgumentException if the argument is incorrect */ public void setEscapedValue(String token) { // update internalSetEscapedValue(String) if this is changed if (token == null) { throw new IllegalArgumentException("Token cannot be null"); //$NON-NLS-1$ } Scanner scanner = this.ast.scanner; char[] source = token.toCharArray(); scanner.setSource(source); scanner.resetTo(0, source.length); try { int tokenType = scanner.getNextToken(); switch(tokenType) { case TerminalTokens.TokenNameStringLiteral: break; default: throw new IllegalArgumentException("Invalid string literal : >" + token + "<"); //$NON-NLS-1$//$NON-NLS-2$ } } catch(InvalidInputException e) { throw new IllegalArgumentException("Invalid string literal : >" + token + "<");//$NON-NLS-1$//$NON-NLS-2$ } preValueChange(ESCAPED_VALUE_PROPERTY); this.escapedValue = token; postValueChange(ESCAPED_VALUE_PROPERTY); }
private boolean commentStartsBlock(int start, int end) { this.localScanner.resetTo(start, end); try { if (this.localScanner.getNextToken() == TerminalTokens.TokenNameLBRACE) { switch(this.localScanner.getNextToken()) { case TerminalTokens.TokenNameCOMMENT_BLOCK : case TerminalTokens.TokenNameCOMMENT_JAVADOC : case TerminalTokens.TokenNameCOMMENT_LINE : return true; } } } catch(InvalidInputException e) { // ignore } return false; }
/** * This method is used to set the right end position for expression * statement. The actual AST nodes don't include the trailing semicolon. * This method fixes the length of the corresponding node. */ protected void retrieveColonPosition(ASTNode node) { int start = node.getStartPosition(); int length = node.getLength(); int end = start + length; this.scanner.resetTo(end, this.compilationUnitSourceLength); try { int token; while ((token = this.scanner.getNextToken()) != TerminalTokens.TokenNameEOF) { switch(token) { case TerminalTokens.TokenNameCOLON: node.setSourceRange(start, this.scanner.currentPosition - start); return; } } } catch(InvalidInputException e) { // ignore } }
protected int retrieveSemiColonPosition(Expression node) { int start = node.getStartPosition(); int length = node.getLength(); int end = start + length; this.scanner.resetTo(end, this.compilationUnitSourceLength); try { int token; while ((token = this.scanner.getNextToken()) != TerminalTokens.TokenNameEOF) { switch(token) { case TerminalTokens.TokenNameSEMICOLON: return this.scanner.currentPosition - 1; } } } catch(InvalidInputException e) { // ignore } return -1; }
/** * This method is used to retrieve the position just before the left bracket. * @return int the dimension found, -1 if none */ protected int retrieveEndOfElementTypeNamePosition(int start, int end) { this.scanner.resetTo(start, end); try { int token; while ((token = this.scanner.getNextToken()) != TerminalTokens.TokenNameEOF) { switch(token) { case TerminalTokens.TokenNameIdentifier: case TerminalTokens.TokenNamebyte: case TerminalTokens.TokenNamechar: case TerminalTokens.TokenNamedouble: case TerminalTokens.TokenNamefloat: case TerminalTokens.TokenNameint: case TerminalTokens.TokenNamelong: case TerminalTokens.TokenNameshort: case TerminalTokens.TokenNameboolean: return this.scanner.currentPosition - 1; } } } catch(InvalidInputException e) { // ignore } return -1; }
/** * This method is used to retrieve the start position of the block. * @return int the dimension found, -1 if none */ protected int retrieveIdentifierEndPosition(int start, int end) { this.scanner.resetTo(start, end); try { int token; while ((token = this.scanner.getNextToken()) != TerminalTokens.TokenNameEOF) { switch(token) { case TerminalTokens.TokenNameIdentifier://110 return this.scanner.getCurrentTokenEndPosition(); } } } catch(InvalidInputException e) { // ignore } return -1; }
protected boolean parseThrows() { int start = this.scanner.currentPosition; try { Object typeRef = parseQualifiedName(true); if (this.abort) return false; // May be aborted by specialized parser if (typeRef == null) { if (this.reportProblems) this.sourceParser.problemReporter().javadocMissingThrowsClassName(this.tagSourceStart, this.tagSourceEnd, this.sourceParser.modifiers); } else { return pushThrowName(typeRef); } } catch (InvalidInputException ex) { if (this.reportProblems) this.sourceParser.problemReporter().javadocInvalidThrowsClass(start, getTokenEndPosition()); } return false; }
/** * Reads the next token. * @param ignoreComments If set, comments will be overread * @return Return the token id. * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE) * or a lexical error was detected while scanning (code LEXICAL_ERROR) */ public int readNext(boolean ignoreComments) throws CoreException { int curr= 0; do { try { curr= fScanner.getNextToken(); if (curr == ITerminalSymbols.TokenNameEOF) { throw new CoreException(createError(END_OF_FILE, "End Of File", null)); //$NON-NLS-1$ } } catch (InvalidInputException e) { throw new CoreException(createError(LEXICAL_ERROR, e.getMessage(), e)); } } while (ignoreComments && isComment(curr)); return curr; }
/** * Reads the next token. * @param ignoreComments If set, comments will be overread. * @return Return the token id. * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE) * or a lexical error was detected while scanning (code LEXICAL_ERROR) */ private int readNextWithEOF(boolean ignoreComments) throws CoreException { int curr= 0; do { try { curr= fScanner.getNextToken(); } catch (InvalidInputException e) { throw new CoreException(createError(LEXICAL_ERROR, e.getMessage(), e)); } } while (ignoreComments && isComment(curr)); return curr; }
private boolean isValidComment(String template) { IScanner scanner = ToolFactory.createScanner(true, false, false, false); scanner.setSource(template.toCharArray()); try { int next = scanner.getNextToken(); while (TokenScanner.isComment(next)) { next = scanner.getNextToken(); } return next == ITerminalSymbols.TokenNameEOF; } catch (InvalidInputException e) { } return false; }
private void check(RefactoringStatus result, Selection selection, IScanner scanner, int start, int end) { char[] characters = scanner.getSource(); selection = adjustSelection(characters, selection, end); scanner.resetTo(start, end); int token = 0; try { loop: while (token != ITerminalSymbols.TokenNameEOF) { token = scanner.getNextToken(); switch (token) { case ITerminalSymbols.TokenNameCOMMENT_LINE: case ITerminalSymbols.TokenNameCOMMENT_BLOCK: case ITerminalSymbols.TokenNameCOMMENT_JAVADOC: if (checkStart(scanner, selection.getOffset())) { result.addFatalError(RefactoringCoreMessages.CommentAnalyzer_starts_inside_comment); break loop; } if (checkEnd(scanner, selection.getInclusiveEnd())) { result.addFatalError(RefactoringCoreMessages.CommentAnalyzer_ends_inside_comment); break loop; } break; } } } catch (InvalidInputException e) { result.addFatalError(RefactoringCoreMessages.CommentAnalyzer_internal_error); } }
/** * Reads the next token. * * @param ignoreComments If set, comments will be overread * @return Return the token id. * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE) or * a lexical error was detected while scanning (code LEXICAL_ERROR) */ public int readNext(boolean ignoreComments) throws CoreException { int curr = 0; do { try { curr = fScanner.getNextToken(); if (curr == ITerminalSymbols.TokenNameEOF) { throw new CoreException(createError(END_OF_FILE, "End Of File", null)); // $NON-NLS-1$ } } catch (InvalidInputException e) { throw new CoreException(createError(LEXICAL_ERROR, e.getMessage(), e)); } } while (ignoreComments && isComment(curr)); return curr; }
/** * Reads the next token. * * @param ignoreComments If set, comments will be overread. * @return Return the token id. * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE) or * a lexical error was detected while scanning (code LEXICAL_ERROR) */ private int readNextWithEOF(boolean ignoreComments) throws CoreException { int curr = 0; do { try { curr = fScanner.getNextToken(); } catch (InvalidInputException e) { throw new CoreException(createError(LEXICAL_ERROR, e.getMessage(), e)); } } while (ignoreComments && isComment(curr)); return curr; }