What's wrong with this Lucene TokenFilter?

Disclaimer: I've been coding for 36 of the last 41 hours. I have a headache. And I can't figure out why this combining TokenFilter is returning 2 tokens, both the first token from the source stream.

public class TokenCombiner extends TokenFilter {

  /*
   * Recombines all tokens back into a single token using the specified delimiter.
   */
  public TokenCombiner(TokenStream in, int delimiter) {
    super(in);
    this.delimiter = delimiter;
  }
  int delimiter;


  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);


  private boolean firstToken = true;
  int startOffset = 0;
  @Override
  public final boolean incrementToken() throws IOException {
    while (true){ 
        boolean eos = input.incrementToken(); //We have to process tokens even if they return end of file.
        CharTermAttribute token = input.getAttribute(CharTermAttribute.class);
        if (eos && token.length() == 0) break; //Break early to avoid extra whitespace.
        if (firstToken){
            startOffset = input.getAttribute(OffsetAttribute.class).startOffset();
            firstToken = false;

        }else{
            termAtt.append(Character.toString((char)delimiter));
        }
        termAtt.append(token);
        if (eos) break;
    }
    offsetAtt.setOffset(startOffset, input.getAttribute(OffsetAttribute.class).endOffset());
    return false;
  }开发者_高级运维

  @Override
  public void reset() throws IOException {
    super.reset();
    firstToken = true;
    startOffset = 0;
  }
}

I think the fundamental problem here, is that you must realize both TokenCombiner and the producer it consumes (input) share and reuse the same attributes! So token == termAtt always (try adding an assert!).

Man, that sucks if you have been coding for 36 hours on a weekend... try this:

public class TokenCombiner extends TokenFilter {
  private final StringBuilder sb = new StringBuilder();
  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
  private final char separator;
  private boolean consumed; // true if we already consumed

  protected TokenCombiner(TokenStream input, char separator) {
    super(input);
    this.separator = separator;
  }

  @Override
  public final boolean incrementToken() throws IOException {
    if (consumed) {
      return false; // don't call input.incrementToken() after it returns false
    }
    consumed = true;

    int startOffset = 0;
    int endOffset = 0;

    boolean found = false; // true if we actually consumed any tokens
    while (input.incrementToken()) {
      if (!found) {
        startOffset = offsetAtt.startOffset();
        found = true;
      }
      sb.append(termAtt);
      sb.append(separator);
      endOffset = offsetAtt.endOffset();
    }

    if (found) {
      assert sb.length() > 0; // always: because we append separator
      sb.setLength(sb.length() - 1);
      clearAttributes();
      termAtt.setEmpty().append(sb);
      offsetAtt.setOffset(startOffset, endOffset);
      return true;
    } else {
      return false;
    }
  }

  @Override
  public void reset() throws IOException {
    super.reset();
    sb.setLength(0);
    consumed = false;
  }
}