How to use

BasicTokenIterator

in

org.apache.http.message

  /**
   * Creates a token iterator from a header iterator.
   * This method can be overridden to replace the implementation of
   * the token iterator.
   *
   * @param hit       the header iterator
   *
   * @return  the token iterator
   */
  protected TokenIterator createTokenIterator(HeaderIterator hit) {
    return new BasicTokenIterator(hit);
  }
}

} else {
  from = findTokenSeparator(from);
int start = findTokenStart(from);
if (start < 0) {
  this.currentToken = null;
int end = findTokenEnd(start);
this.currentToken = createToken(this.currentHeader, start, end);
return end;

/**
 * Creates a new instance of {@link BasicTokenIterator}.
 *
 * @param headerIterator    the iterator for the headers to tokenize
 */
public BasicTokenIterator(final HeaderIterator headerIterator) {
  if (headerIterator == null) {
    throw new IllegalArgumentException
      ("Header iterator must not be null.");
  }
  this.headerIt = headerIterator;
  this.searchPos = findNext(-1);
}

while (!found && (from < to)) {
  final char ch = this.currentHeader.charAt(from);
  if (isTokenSeparator(ch)) {
    found = true;
  } else if (isWhitespace(ch)) {
    from++;
  } else if (isTokenChar(ch)) {
    throw new ParseException
      ("Tokens without separator (pos " + from +

/**
 * Determines the ending position of the current token.
 * This method will not leave the current header value,
 * since the end of the header value is a token boundary.
 *
 * @param from      the position of the first character of the token
 *
 * @return  the position after the last character of the token.
 *          The behavior is undefined if <code>from</code> does not
 *          point to a token character in the current header value.
 */
protected int findTokenEnd(int from) {
  if (from < 0) {
    throw new IllegalArgumentException
      ("Token start position must not be negative: " + from);
  }
  final int to = this.currentHeader.length();
  int end = from+1;
  while ((end < to) && isTokenChar(this.currentHeader.charAt(end))) {
    end++;
  }
  return end;
}

/**
 * Checks whether a character is a valid token character.
 * Whitespace, control characters, and HTTP separators are not
 * valid token characters. The HTTP specification (RFC 2616, section 2.2)
 * defines tokens only for the US-ASCII character set, this
 * method extends the definition to other character sets.
 *
 * @param ch        the character to check
 *
 * @return  <code>true</code> if the character is a valid token start,
 *          <code>false</code> otherwise
 */
protected boolean isTokenChar(char ch) {
  // common sense extension of ALPHA + DIGIT
  if (Character.isLetterOrDigit(ch))
    return true;
  // common sense extension of CTL
  if (Character.isISOControl(ch))
    return false;
  // no common sense extension for this
  if (isHttpSeparator(ch))
    return false;
  // RFC 2616, section 2.2 defines a token character as
  // "any CHAR except CTLs or separators". The controls
  // and separators are included in the checks above.
  // This will yield unexpected results for Unicode format characters.
  // If that is a problem, overwrite isHttpSeparator(char) to filter
  // out the false positives.
  return true;
}

if (isTokenSeparator(ch) || isWhitespace(ch)) {
} else if (isTokenChar(this.currentHeader.charAt(from))) {

/**
 * Determines the ending position of the current token.
 * This method will not leave the current header value,
 * since the end of the header value is a token boundary.
 *
 * @param from      the position of the first character of the token
 *
 * @return  the position after the last character of the token.
 *          The behavior is undefined if <code>from</code> does not
 *          point to a token character in the current header value.
 */
protected int findTokenEnd(int from) {
  if (from < 0) {
    throw new IllegalArgumentException
      ("Token start position must not be negative: " + from);
  }
  final int to = this.currentHeader.length();
  int end = from+1;
  while ((end < to) && isTokenChar(this.currentHeader.charAt(end))) {
    end++;
  }
  return end;
}

/**
 * Checks whether a character is a valid token character.
 * Whitespace, control characters, and HTTP separators are not
 * valid token characters. The HTTP specification (RFC 2616, section 2.2)
 * defines tokens only for the US-ASCII character set, this
 * method extends the definition to other character sets.
 *
 * @param ch        the character to check
 *
 * @return  <code>true</code> if the character is a valid token start,
 *          <code>false</code> otherwise
 */
protected boolean isTokenChar(char ch) {
  // common sense extension of ALPHA + DIGIT
  if (Character.isLetterOrDigit(ch))
    return true;
  // common sense extension of CTL
  if (Character.isISOControl(ch))
    return false;
  // no common sense extension for this
  if (isHttpSeparator(ch))
    return false;
  // RFC 2616, section 2.2 defines a token character as
  // "any CHAR except CTLs or separators". The controls
  // and separators are included in the checks above.
  // This will yield unexpected results for Unicode format characters.
  // If that is a problem, overwrite isHttpSeparator(char) to filter
  // out the false positives.
  return true;
}

while (!found && (from < to)) {
  final char ch = this.currentHeader.charAt(from);
  if (isTokenSeparator(ch)) {
    found = true;
  } else if (isWhitespace(ch)) {
    from++;
  } else if (isTokenChar(ch)) {
    throw new ParseException
      ("Tokens without separator (pos " + from +

} else {
  from = findTokenSeparator(from);
final int start = findTokenStart(from);
if (start < 0) {
  this.currentToken = null;
final int end = findTokenEnd(start);
this.currentToken = createToken(this.currentHeader, start, end);
return end;

/**
 * Creates a token iterator from a header iterator.
 * This method can be overridden to replace the implementation of
 * the token iterator.
 *
 * @param hit       the header iterator
 *
 * @return  the token iterator
 */
protected TokenIterator createTokenIterator(final HeaderIterator hit) {
  return new BasicTokenIterator(hit);
}

/**
 * Obtains the next token from this iteration.
 *
 * @return  the next token in this iteration
 *
 * @throws NoSuchElementException   if the iteration is already over
 * @throws ParseException   if an invalid header value is encountered
 */
public String nextToken()
  throws NoSuchElementException, ParseException {
  if (this.currentToken == null) {
    throw new NoSuchElementException("Iteration already finished.");
  }
  final String result = this.currentToken;
  // updates currentToken, may trigger ParseException:
  this.searchPos = findNext(this.searchPos);
  return result;
}

/**
 * Determines the ending position of the current token.
 * This method will not leave the current header value,
 * since the end of the header value is a token boundary.
 *
 * @param from      the position of the first character of the token
 *
 * @return  the position after the last character of the token.
 *          The behavior is undefined if <code>from</code> does not
 *          point to a token character in the current header value.
 */
protected int findTokenEnd(int from) {
  if (from < 0) {
    throw new IllegalArgumentException
      ("Token start position must not be negative: " + from);
  }
  final int to = this.currentHeader.length();
  int end = from+1;
  while ((end < to) && isTokenChar(this.currentHeader.charAt(end))) {
    end++;
  }
  return end;
}

/**
 * Checks whether a character is a valid token character.
 * Whitespace, control characters, and HTTP separators are not
 * valid token characters. The HTTP specification (RFC 2616, section 2.2)
 * defines tokens only for the US-ASCII character set, this
 * method extends the definition to other character sets.
 *
 * @param ch        the character to check
 *
 * @return  <code>true</code> if the character is a valid token start,
 *          <code>false</code> otherwise
 */
protected boolean isTokenChar(char ch) {
  // common sense extension of ALPHA + DIGIT
  if (Character.isLetterOrDigit(ch))
    return true;
  // common sense extension of CTL
  if (Character.isISOControl(ch))
    return false;
  // no common sense extension for this
  if (isHttpSeparator(ch))
    return false;
  // RFC 2616, section 2.2 defines a token character as
  // "any CHAR except CTLs or separators". The controls
  // and separators are included in the checks above.
  // This will yield unexpected results for Unicode format characters.
  // If that is a problem, overwrite isHttpSeparator(char) to filter
  // out the false positives.
  return true;
}

while (!found && (from < to)) {
  final char ch = this.currentHeader.charAt(from);
  if (isTokenSeparator(ch)) {
    found = true;
  } else if (isWhitespace(ch)) {
    from++;
  } else if (isTokenChar(ch)) {
    throw new ParseException
      ("Tokens without separator (pos " + from +

} else {
  from = findTokenSeparator(from);
final int start = findTokenStart(from);
if (start < 0) {
  this.currentToken = null;
final int end = findTokenEnd(start);
this.currentToken = createToken(this.currentHeader, start, end);
return end;

/**
 * Creates a token iterator from a header iterator.
 * This method can be overridden to replace the implementation of
 * the token iterator.
 *
 * @param hit       the header iterator
 *
 * @return  the token iterator
 */
protected TokenIterator createTokenIterator(final HeaderIterator hit) {
  return new BasicTokenIterator(hit);
}

/**
 * Creates a new instance of {@link BasicTokenIterator}.
 *
 * @param headerIterator    the iterator for the headers to tokenize
 */
public BasicTokenIterator(final HeaderIterator headerIterator) {
  if (headerIterator == null) {
    throw new IllegalArgumentException
      ("Header iterator must not be null.");
  }
  this.headerIt = headerIterator;
  this.searchPos = findNext(-1);
}

/**
 * Determines the ending position of the current token.
 * This method will not leave the current header value,
 * since the end of the header value is a token boundary.
 *
 * @param from      the position of the first character of the token
 *
 * @return  the position after the last character of the token.
 *          The behavior is undefined if <code>from</code> does not
 *          point to a token character in the current header value.
 */
protected int findTokenEnd(int from) {
  if (from < 0) {
    throw new IllegalArgumentException
      ("Token start position must not be negative: " + from);
  }
  final int to = this.currentHeader.length();
  int end = from+1;
  while ((end < to) && isTokenChar(this.currentHeader.charAt(end))) {
    end++;
  }
  return end;
}