Java 类org.eclipse.jface.text.rules.WordRule 实例源码

/**
 * A default constructor.
 * @param manager
 */
public TexArgScanner(ColorManager manager) {
    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in argument color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.CURLY_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new WordRule(new TexWord(), commandToken));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}

/**
 * A default constructor.
 * @param manager
 */
public TexOptArgScanner(ColorManager manager) {
    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in argument color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.SQUARE_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new WordRule(new TexWord(), commandToken));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}

public static WordRule buildRule(ImpexRules ruleType, IToken token) {
    WordRule result = null;
    switch (ruleType) {
    case KEYWORD:
        result = buildKeywordRule(token);
        break;
    case VARIABLE:
        result = buildVariableRule('$', token);
        break;
    case REFERENCE:
        result = buildVariableRule('&', token);
        break;
    case SEMICOLON:
        result = buildCharRule(';', token);
        break;
    case COMMA:
        result = buildCharRule(',', token);
        break;
    default:
        result = null;
        break;
    }
    return result;
}

@Override
protected List<IRule> createRules() {
    IToken keywordToken = getToken(Prefs.EDITOR_HTML_KEYWORD_COLOR);
    IToken symbolToken = getToken(Prefs.EDITOR_HTML_SYMBOL_COLOR);
    IToken stringToken = getToken(Prefs.EDITOR_HTML_STRING_COLOR);
    IToken defaultToken = getToken(Prefs.EDITOR_DEFAULT_COLOR);

    WordRule wordRule = new WordRule(new HtmlWordDetector(), defaultToken, true);
    for (String keyword : keywords) {
        wordRule.addWord(keyword, keywordToken);
    }

    WordRule symbolRule = new WordRule(new HtmlSymbolDetector(), defaultToken, true);
    for (String symbol : symbols) {
        symbolRule.addWord(symbol, symbolToken);
    }

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new SingleLineRule("\"", "\"", stringToken, '\\', true));
    rules.add(new SingleLineRule("'", "'", stringToken, '\\', true));
    rules.add(wordRule);
    rules.add(symbolRule);
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    return rules;
}

@Override
protected List<IRule> createRules() {
    IToken keywordToken = getToken(Prefs.EDITOR_MATH_KEYWORD_COLOR);
    IToken symbolToken = getToken(Prefs.EDITOR_MATH_SYMBOL_COLOR);
    IToken commentToken = getToken(Prefs.EDITOR_MATH_COMMENT_COLOR);
    // IToken defaultToken = getToken(Prefs.EDITOR_DEFAULT_COLOR);

    WordRule wordRule = new WordRule(new MathWordDetector(), keywordToken, true);
    WordRule symbolRule = new WordRule(new MathSymbolDetector(), symbolToken, true);

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(wordRule);
    rules.add(symbolRule);
    rules.add(new EndOfLineRule("%", commentToken));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    return rules;
}

@Override
protected List<IRule> createRules() {
    List<IRule> rules = new ArrayList<IRule>();

    Token tagBorder = getToken(IJSXColorConstants.TAG_BORDER);
    Token tagName = getToken(IJSXColorConstants.TAG_NAME);
    Token tagAttributeName = getToken(IJSXColorConstants.TAG_ATTRIBUTE_NAME);
    Token tagAttributeEquals = getToken(IJSXColorConstants.TAG_ATTRIBUTE_EQUALS);
    Token tagAttributeValue = getToken(IJSXColorConstants.TAG_ATTRIBUTE_VALUE);

    rules.add(new SingleLineRule("\"", "\"", tagAttributeValue, '\\'));
    rules.add(new SingleLineRule("'", "'", tagAttributeValue, '\\'));
    rules.add(new SingleLineRule("{", "}", tagAttributeValue, '\\'));
    rules.add(new JSXTagRule(tagName, tagBorder));
    rules.add(new WordRule(new NameDetector(), tagAttributeName));

    // setDefaultReturnToken(token);
    return rules;
}

public void attachWordRule(List<IRule> rules) {
IToken tokenDefault = new Token(new TextAttribute(KSC_DEFAULT));
IToken tokenKeyword = new Token(new TextAttribute(KSC_KEYWORD));
IToken tokenNumber = new Token(new TextAttribute(KSC_NUMBER));
IToken tokenOperator = new Token(new TextAttribute(KSC_OPERATOR));

WordRule wordRule = new WordRule(new KernelSyntaxWordDetector(), tokenDefault, true);
for (String keyword : KSL_KEYWORD) {
    wordRule.addWord(keyword, tokenKeyword);
}
rules.add(wordRule);

KernelSyntaxNumberRule numberRule = new KernelSyntaxNumberRule(tokenNumber);
rules.add(numberRule);

KernelSyntaxOperatorRule opRule = new KernelSyntaxOperatorRule(tokenOperator);
rules.add(opRule);
}

public void attachWordRule(List<IRule> rules) {
IToken tokenDefault = new Token(new TextAttribute(KSC_DEFAULT));
IToken tokenKeyword = new Token(new TextAttribute(KSC_KEYWORD));
IToken tokenNumber = new Token(new TextAttribute(KSC_NUMBER));
IToken tokenOperator = new Token(new TextAttribute(KSC_OPERATOR));

WordRule wordRule = new WordRule(new KernelSyntaxWordDetector(), tokenDefault, true);
for (String keyword : KSL_KEYWORD) {
    wordRule.addWord(keyword, tokenKeyword);
}
rules.add(wordRule);

KernelSyntaxNumberRule numberRule = new KernelSyntaxNumberRule(tokenNumber);
rules.add(numberRule);

KernelSyntaxOperatorRule opRule = new KernelSyntaxOperatorRule(tokenOperator);
rules.add(opRule);
}

public void attachWordRule(List<IRule> rules) {
IToken tokenDefault = new Token(new TextAttribute(KSC_DEFAULT));
IToken tokenKeyword = new Token(new TextAttribute(KSC_KEYWORD));
IToken tokenNumber = new Token(new TextAttribute(KSC_NUMBER));
IToken tokenOperator = new Token(new TextAttribute(KSC_OPERATOR));

WordRule wordRule = new WordRule(new KernelSyntaxWordDetector(), tokenDefault, true);
for (String keyword : KSL_KEYWORD) {
    wordRule.addWord(keyword, tokenKeyword);
}
rules.add(wordRule);

KernelSyntaxNumberRule numberRule = new KernelSyntaxNumberRule(tokenNumber);
rules.add(numberRule);

KernelSyntaxOperatorRule opRule = new KernelSyntaxOperatorRule(tokenOperator);
rules.add(opRule);
}

/**
 * This method is called when the we need to recreate the scanning rules. 
 * It is called when the scanner is created and when colour preferences
 * and task preferences change. 
 */
/* Override */
protected void update()
{
   LinkedList<IRule> rules = new LinkedList<IRule>();

   this.setDefaultReturnToken( getTokenForPreference( Preference.COLOUR_COMMENT ) );

   // Add whitespace matcher
   rules.add( new WhitespaceRule( new WhiteSpaceDetector() ) );

   WordRule taskRule = new WordRule( new WordDetector(), getTokenForPreference( Preference.COLOUR_COMMENT ) );

   // Add all the task tags 
   for( String tag : TaskTag.getTaskTagStrings( 
      PreferenceManager.getPreference( Preference.TASK_TAGS ) ) ) 
   {
      taskRule.addWord( tag, getTokenForPreference( Preference.COLOUR_TASK ) );
   }

   rules.add( taskRule );

   // Set the rules
   this.setRules( rules.toArray( new IRule[rules.size()] ) );   
}

/**
 * XMLTokenScanner
 */
public XMLTokenScanner()
{
    List<IRule> rules = new ArrayList<IRule>();

    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new MultiLineRule("<!--", "-->", createToken(XMLTokenType.COMMENT))); //$NON-NLS-1$ //$NON-NLS-2$
    rules.add(new DocTypeRule(createToken(XMLTokenType.DOCTYPE), false));
    rules.add(new MultiLineRule("<![CDATA[", "]]>", createToken(XMLTokenType.CDATA))); //$NON-NLS-1$ //$NON-NLS-2$
    rules.add(new TagRule("?xml", createToken(XMLTokenType.DECLARATION))); //$NON-NLS-1$
    rules.add(new TagRule("/", createToken(XMLTokenType.END_TAG))); //$NON-NLS-1$
    rules.add(new TagRule(createToken(XMLTokenType.START_TAG)));

    // text
    IToken token = createToken(XMLTokenType.TEXT);
    rules.add(new WordRule(new WordDetector(), token));

    setRules(rules.toArray(new IRule[rules.size()]));
    setDefaultReturnToken(token);
}

/**
 * Create a new javadoc scanner for the given color provider.
 */
public JSDocScanner()
{
    super();

    List<IRule> list = new ArrayList<IRule>();

    // Add rule for tags.
    list.add(new SingleLineRule("<", ">", getToken("text.html.basic"))); //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-1$

    // Add rule for links.
    list.add(new SingleLineRule("{", "}", getToken("markup.underline.link"))); //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-1$

    // Add word rule for keywords.
    IToken keyword = getToken("meta.tag.documentation.js"); //$NON-NLS-1$
    WordRule wordRule = new WordRule(new JSDocWordDetector());
    for (String word : KEYWORDS)
    {
        wordRule.addWord(word, keyword);
    }
    list.add(wordRule);

    setDefaultReturnToken(getToken("comment.block.documentation.js")); //$NON-NLS-1$
    setRules(list.toArray(new IRule[list.size()]));
}

/**
 * createScannerSpecificRules
 * 
 * @return
 */
protected Collection<? extends IRule> createScannerSpecificRules()
{
    List<IRule> rules = new ArrayList<IRule>();
    WordRule wordRule = new WordRule(new KeywordIdentifierDetector(), Token.UNDEFINED);
    wordRule.addWord("em", createToken(CSSTokenType.EMS));
    wordRule.addWord("ex", createToken(CSSTokenType.EXS));
    wordRule.addWord("px", createToken(CSSTokenType.LENGTH));
    wordRule.addWord("cm", createToken(CSSTokenType.LENGTH));
    wordRule.addWord("mm", createToken(CSSTokenType.LENGTH));
    wordRule.addWord("in", createToken(CSSTokenType.LENGTH));
    wordRule.addWord("pt", createToken(CSSTokenType.LENGTH));
    wordRule.addWord("pc", createToken(CSSTokenType.LENGTH));
    wordRule.addWord("deg", createToken(CSSTokenType.ANGLE));
    wordRule.addWord("rad", createToken(CSSTokenType.ANGLE));
    wordRule.addWord("grad", createToken(CSSTokenType.ANGLE));
    wordRule.addWord("ms", createToken(CSSTokenType.TIME));
    wordRule.addWord("s", createToken(CSSTokenType.TIME));
    wordRule.addWord("hz", createToken(CSSTokenType.FREQUENCY));
    wordRule.addWord("khz", createToken(CSSTokenType.FREQUENCY));
    wordRule.addWord("Hz", createToken(CSSTokenType.FREQUENCY));
    wordRule.addWord("kHz", createToken(CSSTokenType.FREQUENCY));
    addWordsToRule(wordRule, FUNCTIONS, CSSTokenType.FUNCTION);
    rules.add(wordRule);
    return rules;
}

@Override
protected List<IRule> createRules() {
    setDefaultReturnToken(getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_VALUE));
    List<IRule> rules= new ArrayList<IRule>();

    // Add rule for arguments.
    IToken token= getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_ARGUMENT);
    rules.add(new ArgumentRule(token));

    // Add word rule for assignment operator.
    token= getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_ASSIGNMENT);
    WordRule wordRule= new WordRule(new AssignmentDetector(), token);
    rules.add(wordRule);

    // Add generic whitespace rule.
    rules.add(new WhitespaceRule(new JavaWhitespaceDetector()));

    return rules;
}

/**
 * Creates the list of <code>IRule</code>.
 * If you have to customize rules, override this method.
 *
 * @return the list of <code>IRule</code>
 */
protected List<IRule> createRules() {
    IToken normal  = new Token(new TextAttribute(new Color(Display.getCurrent(),
            IHTMLColorConstants.FOREGROUND)));
    IToken string  = new Token(new TextAttribute(new Color(Display.getCurrent(),
            IHTMLColorConstants.JAVA_STRING)));
    IToken comment = new Token(new TextAttribute(new Color(Display.getCurrent(),
            IHTMLColorConstants.JAVA_COMMENT)));
    IToken keyword = new Token(new TextAttribute(new Color(Display.getCurrent(),
            IHTMLColorConstants.JAVA_KEYWORD)));
    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new SingleLineRule("\"", "\"", string, '\\'));
    rules.add(new SingleLineRule("'", "'", string, '\\'));
    rules.add(new SingleLineRule("\\//", null, normal));
    rules.add(new EndOfLineRule("//", comment));
    WordRule wordRule = new WordRule(new JavaWordDetector(), normal);
    for (int i = 0; i < KEYWORDS.length; i++) {
        wordRule.addWord(KEYWORDS[i], keyword);
    }
    rules.add(wordRule);
    return rules;
}

@Override
protected List<IRule> createRules() {
    setDefaultReturnToken(getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_VALUE));
    List<IRule> rules= new ArrayList<IRule>();

    // Add rule for arguments.
    IToken token= getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_ARGUMENT);
    rules.add(new ArgumentRule(token));

    // Add word rule for assignment operator.
    token= getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_ASSIGNMENT);
    WordRule wordRule= new WordRule(new AssignmentDetector(), token);
    rules.add(wordRule);

    // Add generic whitespace rule.
    rules.add(new WhitespaceRule(new JavaWhitespaceDetector()));

    return rules;
}

public MinetweakerScanner(ColorManager manager) {
    IToken comments = new Token(new TextAttribute(manager.getColor(ZSReferences.ZS_COMMENT)));
    IToken keywords = new Token(new TextAttribute(manager.getColor(ZSReferences.KEYWORD)));

    IRule[] rules = new IRule[4];
    rules[0] = new EndOfLineRule("#", comments);
    rules[1] = new EndOfLineRule("//", comments);
    rules[2] = new WhitespaceRule(new MinetweakerWhitespaceDetector());

    WordRule wordRule = new WordRule(new MinetweakerWordDetector(), Token.WHITESPACE);
    for(String keyword : ZSReferences.KEYWORDS)
        wordRule.addWord(keyword, keywords);

    rules[3] = wordRule;

    setRules(rules);
}

@Override
protected List<IRule> createRules() {
  List<IRule> rules = new ArrayList<IRule>();
  IToken keyword = this.getToken(GnColorConstants.GN_KEYWORD);
  IToken numberToken = getToken(GnColorConstants.GN_NUMBER);
  IToken comment = this.getToken(GnColorConstants.GN_COMMENT);
  IToken other = this.getToken(GnColorConstants.GN_DEFAULT);

  rules.add(new EndOfLineRule("#", comment));
  rules.add(new WhitespaceRule(new GnWhitespaceDetector()));

  WordRule wordRule = new WordRule(new GnWordDetector(), other);
  for (int i = 0; i < GnCodeUtil.KEYWORDS.length; i++) {
    wordRule.addWord(GnCodeUtil.KEYWORDS[i], keyword);
  }
  rules.add(wordRule);

  NumberRule numberRule = new NumberRule(numberToken);
  rules.add(numberRule);

  this.setDefaultReturnToken(other);
  return rules;
}

private void initScanner() {
    IToken string = new Token(new TextAttribute(getPreferenceColor(JsonPreferenceStore.STRING_COLOR)));
    IToken value = new Token(new TextAttribute(getPreferenceColor(JsonPreferenceStore.VALUE_COLOR)));
    IToken defaultText = new Token(new TextAttribute(getPreferenceColor(JsonPreferenceStore.DEFAULT_COLOR)));
    IToken nullValue = new Token(new TextAttribute(getPreferenceColor(JsonPreferenceStore.NULL_COLOR)));

    List<IRule> rules= new LinkedList<IRule>();

    rules.add(new MultiLineRule(":\"", "\"", value, '\\'));
    rules.add(new MultiLineRule("\"", "\"", string, '\\')); //$NON-NLS-2$ //$NON-NLS-1$
    WordRule wordRule= new WordRule(new JsonWordDetector(), defaultText);
    wordRule.addWord("null", nullValue);
    rules.add(wordRule);
    rules.add(new WhitespaceRule(new JsonWhitespaceDetector()));

    IRule[] result= new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}

private WordRule sqlSyntaxRules() {

        // Define a word rule and add SQL keywords to it.
        WordRule wordRule = new WordRule(new WordDetector(), Token.WHITESPACE, true);
        for (String reservedWord : sqlSyntax.getReservedwords()) {
            wordRule.addWord(reservedWord, new Token(
                    getTextAttribute(prefs, SQLEditorStatementTypes.RESERVED_WORDS)));
        }
        // TODO render unreserved keywords in the same way with reserved
        // keywords, should let user decide via preference
        for (String unreservedWord : sqlSyntax.getUnreservedwords()) {
            wordRule.addWord(unreservedWord, new Token(
                    getTextAttribute(prefs, SQLEditorStatementTypes.UN_RESERVED_WORDS)));
        }

        // Add the SQL datatype names to the word rule.
        for (String datatype : sqlSyntax.getTypes()) {
            wordRule.addWord(datatype, new Token(
                    getTextAttribute(prefs, SQLEditorStatementTypes.TYPES)));
        }

        // Add the SQL function names to the word rule.
        for (String function : sqlSyntax.getFunctions()) {
            wordRule.addWord(function, new Token(
                    getTextAttribute(prefs, SQLEditorStatementTypes.FUNCTIONS)));
        }

        // Add the SQL constants to the word rule.
        for (String constant : sqlSyntax.getConstants()) {
            wordRule.addWord(constant, new Token(
                    getTextAttribute(prefs, SQLEditorStatementTypes.CONSTANTS)));
        }

        return wordRule;
    }

public EiffelCodeScanner(EiffelColorProvider provider) {
    IToken keyword = new Token(new TextAttribute(provider
            .getColor(EiffelColorProvider.KEYWORD)));
    IToken string = new Token(new TextAttribute(provider
            .getColor(EiffelColorProvider.STRING)));
    IToken comment = new Token(new TextAttribute(provider
            .getColor(EiffelColorProvider.SINGLE_LINE_COMMENT)));
    IToken other = new Token(new TextAttribute(provider
            .getColor(EiffelColorProvider.DEFAULT)));

    List<Object> rules = new ArrayList<>();

    //add rule for single line comments.
    rules.add(new EndOfLineRule("--", comment));

    // Add rule for strings and character constants.
    rules.add(new SingleLineRule("\"", "\"", string, '\\'));
    rules.add(new SingleLineRule("'", "'", string, '\\'));

    //Add generic whitespace rule.
    rules.add(new WhitespaceRule(new EiffelWhitespaceDetector()));

    //Add word rule for keywords.
    WordRule wordRule = new WordRule(new EiffelWordDetector(), other);
    for (int i = 0; i < eKeywords.length; i++) {
        wordRule.addWord(eKeywords[i], keyword);
        rules.add(wordRule);

        IRule[] result = new IRule[rules.size()];
        rules.toArray(result);
        setRules(result);
    }
}

public EiffelCodeScanner(EiffelColorProvider provider) {
    IToken keyword = new Token(new TextAttribute(provider
            .getColor(EiffelColorProvider.KEYWORD)));
    IToken string = new Token(new TextAttribute(provider
            .getColor(EiffelColorProvider.STRING)));
    IToken comment = new Token(new TextAttribute(provider
            .getColor(EiffelColorProvider.SINGLE_LINE_COMMENT)));
    IToken other = new Token(new TextAttribute(provider
            .getColor(EiffelColorProvider.DEFAULT)));

    List<Object> rules = new ArrayList<>();

    //add rule for single line comments.
    rules.add(new EndOfLineRule("--", comment));

    // Add rule for strings and character constants.
    rules.add(new SingleLineRule("\"", "\"", string, '\\'));
    rules.add(new SingleLineRule("'", "'", string, '\\'));

    //Add generic whitespace rule.
    rules.add(new WhitespaceRule(new EiffelWhitespaceDetector()));

    //Add word rule for keywords.
    WordRule wordRule = new WordRule(new EiffelWordDetector(), other);
    for (int i = 0; i < Parser.KEYWORDS.length; i++) {
        wordRule.addWord(Parser.KEYWORDS[i], keyword);
        rules.add(wordRule);

        IRule[] result = new IRule[rules.size()];
        rules.toArray(result);
        setRules(result);
    }
}

/**
 * Use the default Eclipse higlighting scheme.
 */
private void createRules() {
    Token keywordToken = new Token(keywordAttribute);
    Token commentToken = new Token(commentAttribute);
    Token docToken = new Token(docAttribute);
    Token stringToken = new Token(stringAttribute);
    Token numberToken = new Token(numberAttribute);
    Token normalToken = new Token(normalAttribute);

    setDefaultReturnToken(normalToken);

    setRules(new IRule[] { new EndOfLineRule("//", commentToken),//$NON-NLS-2$
            new KeywordRule(keywordToken),//$NON-NLS-2$
            new MultiLineRule("/**", "*/", docToken, (char) 0, false), //$NON-NLS-2$
            new MultiLineRule("/*", "*/", commentToken, (char) 0, false), //$NON-NLS-2$
            new SingleLineRule("\"", "\"", stringToken, '\\'), //$NON-NLS-2$
            // Regexp
            new SingleLineRule("/", "/", stringToken, '\\'), //$NON-NLS-2$
            new SingleLineRule("'", "'", stringToken, '\\'), //$NON-NLS-2$
            new WhitespaceRule(new IWhitespaceDetector() {
                public boolean isWhitespace(char c) {
                    return Character.isWhitespace(c);
                }
            }),//$NON-NLS-2$
            new WordRule(new WordDetector(), normalToken),//$NON-NLS-2$
            new NumberRule(numberToken) });
}

/**
 * Creates a BibTeX document scanner
 * 
 * @param provider The color provider for syntax highlighting
 */
public BibCodeScanner(BibColorProvider provider) {

    IToken keyword = new Token(new TextAttribute(provider
            .getColor(BibColorProvider.KEYWORD)));
    IToken comment = new Token(new TextAttribute(provider
            .getColor(BibColorProvider.SINGLE_LINE_COMMENT)));
    IToken other = new Token(new TextAttribute(provider
            .getColor(BibColorProvider.DEFAULT)));

    List rules = new ArrayList();

    // Add rule for single line comments.
    rules.add(new EndOfLineRule("%", comment));

    rules.add(new BibCommandRule(keyword));

    // Add generic whitespace rule.
    rules.add(new WhitespaceRule(new WhitespaceDetector()));

    // Add word rule for keywords, types, and constants.
    WordRule wordRule = new WordRule(new BibWordDetector(), other);
    rules.add(wordRule);

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}

/**
 * A default constructor.
 * @param manager
 */
public TexScanner(ColorManager manager) {
    // A token that defines how to color numbers
    IToken numberToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.TEX_NUMBER),
            null,
            manager.getStyle(ColorManager.TEX_NUMBER_STYLE)));

    // A token that defines how to color command words (\command_word)
    IToken commandToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMAND),
            null,
            manager.getStyle(ColorManager.COMMAND_STYLE)));

    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    // A token that defines how to color special characters (\_, \&, \~ ...)
    IToken specialCharToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.TEX_SPECIAL),
            null,
            manager.getStyle(ColorManager.TEX_SPECIAL_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new TexSpecialCharRule(specialCharToken));
    rules.add(new WordRule(new TexWord(), commandToken));
    rules.add(new NumberRule(numberToken));
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}

public InstructionsRuleScanner(ColorProvider provider) {
    IToken stringToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_STRING_COLOR))));
    IToken commentToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_COMMENT_COLOR))));
    IToken instructToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_INSTRUCT_COLOR)), null, Font.ITALIC));
    IToken definitionsToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_DEF_COLOR)), null, Font.ITALIC));

    List<IRule> rules = Lists.newArrayList();
    // rule for Strings - may spanning multiple lines
    rules.add(new MultiLineRule("\"", "\"", stringToken));
    rules.add(new MultiLineRule("\'", "\'", stringToken));
    rules.add(new EndOfLineRule("#%", instructToken));
    // rule for comments - ended by a line delimiter
    rules.add(new EndOfLineRule("//", commentToken));

    WordRule wordRule = RuleFactory.buildRule(ImpexRules.KEYWORD, null);        
    // rule for instructions
    for (String word : Formatter.INSTRUCTION_CLASS_PROPOSALS) {
        wordRule.addWord(word, instructToken);
    }

    rules.add(wordRule);

    // rule for definitions
    wordRule = RuleFactory.buildRule(ImpexRules.VARIABLE, definitionsToken);
    rules.add(wordRule);
    IRule[] ruleArray = new IRule[rules.size()];
    setRules(rules.toArray(ruleArray));
}

/**
 * Construct the rules
 */
public TLACodeScanner()
{
    TLAColorProvider provider = TLAEditorActivator.getDefault().getTLAColorProvider();

    IToken keyword = new Token(new TextAttribute(provider.getColor(TLAColorProvider.TLA_KEYWORD), null, SWT.BOLD));
    IToken value = new Token(new TextAttribute(provider.getColor(TLAColorProvider.TLA_VALUE)));
    IToken other = new Token(new TextAttribute(provider.getColor(TLAColorProvider.TLA_DEFAULT)));

    List rules = new ArrayList();

    // Add generic whitespace rule.
    // rules.add(new WhitespaceRule(DocumentHelper.getDefaultWhitespaceDetector()));

    // Add word rule for standard words
    WordRule wordRule = new WordRule(DocumentHelper.getDefaultWordDetector(), other);

    // add values
    for (int i = 0; i < ITLAReserveredWords.ALL_VALUES_ARRAY.length; i++)
    {
        wordRule.addWord(ITLAReserveredWords.ALL_VALUES_ARRAY[i], value);
    }

    // add reserved words
    for (int i = 0; i < ITLAReserveredWords.ALL_WORDS_ARRAY.length; i++)
    {
        wordRule.addWord(ITLAReserveredWords.ALL_WORDS_ARRAY[i], keyword);
    }

    rules.add(wordRule);

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}

@Override
protected List<IRule> createRules() {
    IToken keywordToken = getToken(Prefs.EDITOR_DOT_KEYWORD_COLOR);
    IToken attribsToken = getToken(Prefs.EDITOR_DOT_ATTRIBS_COLOR);
    IToken symbolToken = getToken(Prefs.EDITOR_DOT_SYMBOL_COLOR);
    IToken stringToken = getToken(Prefs.EDITOR_DOT_STRING_COLOR);
    IToken commentToken = getToken(Prefs.EDITOR_DOT_COMMENT_COLOR);
    IToken defaultToken = getToken(Prefs.EDITOR_DEFAULT_COLOR);

    WordRule wordRule = new WordRule(new DotWordDetector(), defaultToken, true);
    for (String keyword : keywords) {
        wordRule.addWord(keyword, keywordToken);
    }

    WordRule attribRule = new WordRule(new DotWordDetector(), defaultToken, true);
    for (String attrib : attribs) {
        wordRule.addWord(attrib, attribsToken);
    }

    WordRule symbolRule = new WordRule(new DotSymbolDetector(), defaultToken, true);
    for (String symbol : symbols) {
        symbolRule.addWord(symbol, symbolToken);
    }

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new MultiLineRule("/*", "*/", commentToken, '\\', true));
    rules.add(new SingleLineRule("//", null, commentToken, '\\', true));
    rules.add(new SingleLineRule("\"", "\"", stringToken, '\\', true));
    rules.add(wordRule);
    rules.add(attribRule);
    rules.add(symbolRule);
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    return rules;
}

public VelocityCodeScanner(VelocityColorProvider manager)
{
    List rules = new ArrayList();
    // Add generic whitespace rule
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    // Add word rule for directives
    Token token = (Token) manager.getToken(IColorConstants.DIRECTIVE);
    WordRule wordRule = new WordRule(new DirectiveDetector(), token);
    token = (Token) manager.getToken(IColorConstants.DIRECTIVE);
    // System directives
    String[] directives = Directive.DIRECTIVES;
    for (int i = directives.length - 1; i >= 0; i--)
    {
        wordRule.addWord(directives[i], token);
    }
    // User directives
    Iterator userDirectives = VelocityEditorEnvironment.getParser().getUserDirectives().iterator();
    while (userDirectives.hasNext())
    {
        wordRule.addWord((String) userDirectives.next(), token);
    }
    rules.add(wordRule);
    // Add pattern rule for formal references
    token = (Token) manager.getToken(IColorConstants.REFERENCE);
    rules.add(new PatternRule("$!{", "}", token, (char) 0, true));
    rules.add(new PatternRule("${", "}", token, (char) 0, true));
    // Add pattern rule for shorthand references
    token = (Token) manager.getToken(IColorConstants.REFERENCE);
    rules.add(new WordPatternRule(new IdentifierDetector(), "$!", null, token));
    rules.add(new WordPatternRule(new IdentifierDetector(), "$", null, token));
    // token = new Token(new
    // TextAttribute(aProvider.getColor(IColorConstants.TAG), null,
    // SWT.BOLD));
    // rules.add(new TagRule(token));
    // token = new Token(new
    // TextAttribute(aProvider.getColor(IColorConstants.COMMENT)));
    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}

public HTMLTagScanner(VelocityColorProvider manager)
{
    fDefaultToken = manager.getToken(IColorConstants.HTML_ATTRIBUTE);
    fElementToken = manager.getToken(IColorConstants.HTML_TAG);
    fEndElementToken = manager.getToken(IColorConstants.HTML_ENDTAG);
    IToken string = manager.getToken(IColorConstants.HTML_String);
    Vector rules = new Vector();
    setDefaultReturnToken(fDefaultToken);
    fElementRule = new WordRule(new HTMLElementDetector(), fDefaultToken);
    Token token = (Token) manager.getToken(IColorConstants.DIRECTIVE);
    WordRule wordRule = new WordRule(new DirectiveDetector(), token);
    token = (Token) manager.getToken(IColorConstants.DIRECTIVE);
    // System directives
    String[] directives = Directive.DIRECTIVES;
    for (int i = directives.length - 1; i >= 0; i--)
    {
        wordRule.addWord(directives[i], token);
    }
    rules.add(wordRule);

    rules.add(new PatternRule("\"", "\"", string, '\0', true));
    rules.add(new PatternRule("'", "'", string, '\0', true));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    token = (Token) manager.getToken(IColorConstants.REFERENCE);
    rules.add(new PatternRule("$!{", "}", token, (char) 0, true));
    rules.add(new PatternRule("${", "}", token, (char) 0, true));
    // Add pattern rule for shorthand references
    token = (Token) manager.getToken(IColorConstants.REFERENCE);
    rules.add(new WordPatternRule(new IdentifierDetector(), "$!", null, token));
    rules.add(new WordPatternRule(new IdentifierDetector(), "$", null, token));
    IRule[] result = new IRule[rules.size()];
    rules.copyInto(result);
    setRules(result);
}

public GlslScanner() {
    Activator plugin = Activator.getDefault();
    final WordRule wordRule = GlslScanners.createWordRule();

    final Token keywordToken = new Token(new TextAttribute(plugin.getColor(GlslEditor.KEYWORD_COLOR), null, SWT.BOLD));
    final Token typeToken = new Token(new TextAttribute(plugin.getColor(GlslEditor.TYPE_COLOR)));
    final Token qualifierToken = new Token(new TextAttribute(plugin.getColor(GlslEditor.QUALIFIER_COLOR), null, SWT.BOLD));
    final Token functionToken = new Token(new TextAttribute(plugin.getColor(GlslEditor.FUNCTION_COLOR)));
    final Token builtInVariableToken = new Token(new TextAttribute(plugin.getColor(GlslEditor.BUILT_IN_VARIABLES_COLOR)));
    final Token commentToken = new Token(new TextAttribute(plugin.getColor(GlslEditor.COMMENTS_COLOR)));

    List<IRule> rules = new ArrayList<>();

    // Rules which affect entire lines at once
    rules.add(new SingleLineRule("//", null, commentToken, '\0', true, false));

    // Rules which don't affect entire lines
    rules.add(new SingleLineRule("/*", "*/", commentToken));
    rules.add(wordRule);

    GlslScanners.addToWordRule(wordRule, Glsl.KEYWORDS, keywordToken);
    GlslScanners.addToWordRule(wordRule, Glsl.TYPES, typeToken);
    GlslScanners.addToWordRule(wordRule, Glsl.QUALIFIERS, qualifierToken);
    GlslScanners.addToWordRule(wordRule, Glsl.FUNCTIONS, functionToken);
    GlslScanners.addToWordRule(wordRule, Glsl.VARIABLES, builtInVariableToken);

    rules.add(new WhitespaceRule(new IWhitespaceDetector() {
        @Override
        public boolean isWhitespace(char c) {
           return Character.isWhitespace(c);
        }
       }));

    IRule[] rulesArray = new IRule[rules.size()];
    for (int i = 0; i < rulesArray.length; i++) {
        rulesArray[i] = rules.get(i);
    }

    setRules(rulesArray);
}

public static final WordRule createWordRule() {
    return new WordRule(new IWordDetector() {

        @Override
        public boolean isWordStart(char c) {
            return c != ' ' && Character.isJavaIdentifierStart(c);
        }

        @Override
        public boolean isWordPart(char arg0) {
            return Character.isJavaIdentifierPart(arg0);
        }
    }, Token.WHITESPACE);
}

public static final RuleBasedScanner createVariableDeclarationScanner() {
    final Token qualifierToken = new Token(new TextAttribute(
        Activator.getDefault().getColor(GlslEditor.QUALIFIER_COLOR)
    ));
    final Token typeToken = new Token(new TextAttribute(
        Activator.getDefault().getColor(GlslEditor.TYPE_COLOR)
    ));
    final Token builtInVariableToken = new Token(new TextAttribute(
        Activator.getDefault().getColor(GlslEditor.BUILT_IN_VARIABLES_COLOR)
    ));
    final Token functionToken = new Token(new TextAttribute(
        Activator.getDefault().getColor(GlslEditor.FUNCTION_COLOR)
    ));

    WordRule wordRule = createWordRule();
    addToWordRule(wordRule, Glsl.QUALIFIERS, qualifierToken);
    addToWordRule(wordRule, Glsl.TYPES, typeToken);
    addToWordRule(wordRule, Glsl.VARIABLES, builtInVariableToken);
    addToWordRule(wordRule, Glsl.FUNCTIONS, functionToken);

    RuleBasedScanner scanner = new RuleBasedScanner();
    scanner.setRules(new IRule[] {
        wordRule,
        // TODO Variable highlighting
    });

    return scanner;
}

/**
 * Use the default Eclipse higlighting scheme.
 */
private void createRules() {
  Token keywordToken = new Token(
      new TextAttribute(EditorColors.getColor(new RGB(127, 0, 85)), null, SWT.BOLD));

  commentToken = new Token(commentAttribute);

  Token jsDocToken = new Token(jsDocAttribute);

  Token stringToken = new Token(
      new TextAttribute(EditorColors.getColor(new RGB(42, 0, 255)), null, SWT.NORMAL));

  RGB blackRgb = new RGB(0, 0, 0);

  Token numberToken = new Token(
      new TextAttribute(EditorColors.getColor(blackRgb), null, SWT.NORMAL));

  Token normalToken = new Token(
      new TextAttribute(EditorColors.getColor(blackRgb), null, SWT.NORMAL));
  setDefaultReturnToken(normalToken);

  setRules(new IRule[] {
      new EndOfLineRule("//", commentToken), //$NON-NLS-1$
      new KeywordRule(keywordToken),
      new MultiLineRule("/**", "*/", jsDocToken, (char) 0, false),  //$NON-NLS-1$ //$NON-NLS-2$
      new MultiLineRule("/*", "*/", commentToken, (char) 0, false),  //$NON-NLS-1$ //$NON-NLS-2$
      new SingleLineRule("\"", "\"", stringToken, '\\'), //$NON-NLS-1$ //$NON-NLS-2$
      // Regexp
      new SingleLineRule("/", "/", stringToken, '\\'), //$NON-NLS-1$ //$NON-NLS-2$
      new SingleLineRule("'", "'", stringToken, '\\'), //$NON-NLS-1$ //$NON-NLS-2$
      new WhitespaceRule(new IWhitespaceDetector() {
        public boolean isWhitespace(char c) {
          return Character.isWhitespace(c);
        }
      }),

      new WordRule(new WordDetector(), normalToken),
      new NumberRule(numberToken),
  });
}

public XMLScanner()
{
    IRule[] rules = new IRule[4];
    rules[0] = new WhitespaceRule(new WhitespaceDetector());
    rules[1] = new EntityRule(createToken(CONSTANT_CHARACTER_ENTITY_XML));
    // non-entity ampersands should be marked as invalid
    rules[2] = new SingleCharacterRule('&', createToken(INVALID_ILLEGAL_BAD_AMPERSAND_XML));
    rules[3] = new WordRule(new WordDetector(), createToken(TEXT));
    setRules(rules);
    setDefaultReturnToken(createToken(TEXT));
}

public XMLTagScanner()
{
    List<IRule> rules = new ArrayList<IRule>();

    // Add rule for double quotes
    rules.add(new MultiLineRule("\"", "\"", doubleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$
    rules.add(new BreakingMultiLineRule("\"", "\"", QUOTED_STRING_BREAKS, doubleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$

    // Add a rule for single quotes
    rules.add(new MultiLineRule("'", "'", singleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$
    rules.add(new BreakingMultiLineRule("'", "'", QUOTED_STRING_BREAKS, singleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$

    // Add generic whitespace rule.
    rules.add(new WhitespaceRule(new WhitespaceDetector()));

    // Tags
    WordRule tagWordRule = new TagWordRule(new TagNameWordDetector(), createToken(XMLTokenType.TAG_NAME), true);
    rules.add(tagWordRule);

    // Attributes
    WordRule attributeWordRule = new WordRule(new AttributeNameWordDetector(), createToken(XMLTokenType.ATTRIBUTE),
            true);
    rules.add(attributeWordRule);

    rules.add(new MultiCharacterRule("</", createToken(XMLTokenType.START_TAG))); //$NON-NLS-1$
    rules.add(new MultiCharacterRule("/>", createToken(XMLTokenType.TAG_SELF_CLOSE))); //$NON-NLS-1$

    CharacterMapRule charsRule = new CharacterMapRule();
    charsRule.add('<', createToken(XMLTokenType.START_TAG));
    charsRule.add('>', createToken(XMLTokenType.END_TAG));
    charsRule.add('=', equalToken);
    rules.add(charsRule);

    setRules(rules.toArray(new IRule[rules.size()]));
    setDefaultReturnToken(createToken(XMLTokenType.TEXT));
}

/**
 * XMLAttributeScanner
 */
XMLAttributeScanner()
{
    List<IRule> rules = new ArrayList<IRule>();

    rules.add(new WhitespaceRule(new WhitespaceDetector()));

    // attribute values
    rules.add(new MultiLineRule("\"", "\"", createToken(XMLTokenType.VALUE), '\\')); //$NON-NLS-1$ //$NON-NLS-2$
    rules.add(new MultiLineRule("'", "'", createToken(XMLTokenType.VALUE), '\\')); //$NON-NLS-1$ //$NON-NLS-2$

    // attribute names
    rules.add(new XMLAttributeRule(createToken(XMLTokenType.ATTRIBUTE)));

    // tag name
    rules.add(new WordRule(new DTDNameDetector(), createToken(XMLTokenType.OTHER), true));

    // special characters
    CharacterMapRule characterRule = new CharacterMapRule();
    characterRule.add('<', createToken(XMLTokenType.OTHER));
    characterRule.add('>', createToken(XMLTokenType.OTHER));
    characterRule.add('=', createToken(XMLTokenType.OTHER));
    rules.add(characterRule);

    setRules(rules.toArray(new IRule[rules.size()]));

    setDefaultReturnToken(createToken(XMLTokenType.OTHER));
}

/**
 * addWordsToRule
 * 
 * @param wordRule
 * @param words
 * @param tokenType
 */
private void addWordsToRule(WordRule wordRule, String[] words, CSSTokenType tokenType)
{
    IToken token = createToken(tokenType);

    for (String word : words)
    {
        wordRule.addWord(word, token);
    }
}

/**
 * createAtWordsRule
 * 
 * @return
 */
protected WordRule createAtWordsRule()
{
    WordRule atRule = new WordRule(new IdentifierWithPrefixDetector('@'), createToken(CSSTokenType.AT_RULE));

    atRule.addWord("@import", createToken(CSSTokenType.IMPORT));
    atRule.addWord("@page", createToken(CSSTokenType.PAGE));
    atRule.addWord(KEYWORD_MEDIA, createToken(CSSTokenType.MEDIA_KEYWORD));
    atRule.addWord("@charset", createToken(CSSTokenType.CHARSET));
    atRule.addWord("@font-face", createToken(CSSTokenType.FONTFACE));
    atRule.addWord("@namespace", createToken(CSSTokenType.NAMESPACE));

    return atRule;
}

/**
 * createAtWordsRule
 * 
 * @return
 */
private WordRule createAtWordsRule()
{
    WordRule atRule = new WordRule(new IdentifierWithPrefixDetector('@'), createToken(CSSTokenType.AT_RULE), true);

    atRule.addWord("@import", createToken(CSSTokenType.IMPORT));
    atRule.addWord("@page", createToken(CSSTokenType.PAGE));
    atRule.addWord("@media", createToken(CSSTokenType.MEDIA_KEYWORD));
    atRule.addWord("@charset", createToken(CSSTokenType.CHARSET));
    atRule.addWord("@font-face", createToken(CSSTokenType.FONTFACE));
    atRule.addWord("@namespace", createToken(CSSTokenType.NAMESPACE));

    return atRule;
}