Lucene中的Tokenizer,TokenFilter学习

java.io.Reader -> com.chenlb.mmseg4j.solr.MMSegTokenizer -> SynonymFilter -> StopFilter -> WordDelimiterFilter -> LowerCaseFilter -> RemoveDuplicatesTokenFilter

public class SynonymFilter extends TokenFilter {

    private static final String TOKEN_TYPE_SYNOnYM= "SYNONYM";

    private Stack synonymStack;
    private SynonymEngine synonymEngine;
    private AttributeSource.State current;
    private final CharTermAttribute bytesTermAttribute;
    private final PositionIncrementAttribute positionIncrementAttribute;

    /**
     * Construct a token stream filtering the given input.
     *
     * @param input
     */
    protected SynonymFilter(TokenStream input, SynonymEngine synonymEngine) {
        super(input);
        this.synOnymEngine= synonymEngine;
        synOnymStack= new Stack<>();

        this.bytesTermAttribute = addAttribute(CharTermAttribute.class);
        this.positiOnIncrementAttribute= addAttribute(PositionIncrementAttribute.class);
    }

    @Override
    public boolean incrementToken() throws IOException {
        if (!synonymStack.isEmpty()) {
            String syn = synonymStack.pop();
            restoreState(current);

//            bytesTermAttribute.setBytesRef(new BytesRef(syn.getBytes()));
//            bytesTermAttribute.resizeBuffer(0);
            bytesTermAttribute.append(syn);

            positionIncrementAttribute.setPositionIncrement(0);
            return true;
        }

        if (!input.incrementToken()) {
            return false;
        }

        if (addAliasesToStack()) {
            current = captureState();
        }

        return true;
    }

    private boolean addAliasesToStack() throws IOException {
        String[] synOnyms= synonymEngine.getSynonyms(bytesTermAttribute.toString());
        if (synOnyms== null) {
            return false;
        }
        for (String synonym : synonyms) {
            synonymStack.push(synonym);
        }
        return true;
    }
}

public class SynonymAnalyzer extends Analyzer {
    @Override
    protected TokenStreamComponents createComponents(String fieldName) {
        StandardTokenizer source = new StandardTokenizer();
        return new TokenStreamComponents(source, new SynonymFilter(new StopFilter(new LowerCaseFilter(source),
                new CharArraySet(StopAnalyzer.ENGLISH_STOP_WORDS_SET, true)), new TestSynonymEngine()));
    }
}

public interface SynonymEngine {
    String[] getSynonyms(String s) throws IOException;
}

public class TestSynonymEngine implements SynonymEngine {

    public static final Map map = new HashMap<>();

    static {
        map.put("quick", new String[]{"fast", "speedy"});
    }

    @Override
    public String[] getSynonyms(String s) throws IOException {
        return map.get(s);
    }
}

public static void main(String[] args) throws IOException {
        SynonymAnalyzer analyzer = new SynonymAnalyzer();
        TokenStream tokenStream = analyzer.tokenStream("contents", new StringReader("The quick brown fox"));
        tokenStream.reset();

        CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
        OffsetAttribute offsetAttribute = tokenStream.addAttribute(OffsetAttribute.class);
        PositionIncrementAttribute positiOnIncrementAttribute=
                tokenStream.addAttribute(PositionIncrementAttribute.class);
        TypeAttribute typeAttribute = tokenStream.addAttribute(TypeAttribute.class);

        int position = 0;
        while (tokenStream.incrementToken()) {
            int positiOnIncrement= positionIncrementAttribute.getPositionIncrement();
            if (positionIncrement > 0) {
                position += positionIncrement;
                System.out.println();
                System.out.print(position + " : ");
            }

            System.out.printf("[%s : %d ->  %d : %s]", charTermAttribute.toString(), offsetAttribute.startOffset(), offsetAttribute.endOffset(),
                    typeAttribute.type());
        }

2 : [quick : 4 ->  9 : ][quickspeedy : 4 ->  9 : ][quickfast : 4 ->  9 : ]
3 : [brown : 10 ->  15 : ]
4 : [fox : 16 ->  19 : ]

String line = null;
    while ((line = in.readLine()) != null) {
      if (line.length() == 0 || line.charAt(0) == '#') {
        continue; // ignore empty lines and comments
      }

      // TODO: we could process this more efficiently.
      String sides[] = split(line, "=>");
      if (sides.length > 1) { // explicit mapping
        if (sides.length != 2) {
          throw new IllegalArgumentException("more than one explicit mapping specified on the same line");
        }
        String inputStrings[] = split(sides[0], ",");
        CharsRef[] inputs = new CharsRef[inputStrings.length];
        for (int i = 0; i    
  

@Override
  protected TokenStreamComponents createComponents(String fieldName) {
    Tokenizer tk = tokenizer.create();
    TokenStream ts = tk;
    for (TokenFilterFactory filter : filters) {
      ts = filter.create(ts);
    }
    return new TokenStreamComponents(tk, ts);
  }

@Override
  public TokenStream create(TokenStream input) {
    // if the fst is null, it means there's actually no synonyms... just return the original stream
    // as there is nothing to do here.
    return map.fst == null ? input : new SynonymFilter(input, map, ignoreCase);
  }