Complete Program

The complete program is divided into three files, one for each class:

• WebScannerTester.java
• MyPushbackReader.java
• WebScanner.java

Copy and paste the source for each class into its own file.

// ------ file: WebScannerTester.java
//
import java.io.*;

class WebScannerTester
{
  public static void main ( String[] args ) throws IOException
  {
    FileReader fr;
    String     token;
    WebScanner wbs;

    if ( args.length != 1 )
    {
      System.out.println("tester inputFile");
      System.exit( -1 );
    }

    fr = new FileReader( args[0] );

    wbs = new WebScanner( fr );

    while ( (token = wbs.read()) != null )
    {
      System.out.println( ":" + token );
    }
  }
}
//
// --- end of WebScannerTester.java

// --- file: MyPushbackReader.java
//
import java.io.* ;

class MyPushbackReader extends BufferedReader
{
  private  int     pushBack = ' ';
  private  boolean isPushBack = false;

  public MyPushbackReader(Reader in)
  {
    super( in );
  }

  public int read() throws IOException
  {
    int chr;

    if ( isPushBack )
    {
      isPushBack = false;
      chr = pushBack;
      return chr;
    }

    chr = super.read();

    return chr;
  }

  //  "return" a char to the input stream.
  public void unread( char c ) throws IOException
  {
    if ( isPushBack ) throw new IOException("Pushback buffer is full");
    pushBack   = (int)c;
    isPushBack = true;
  }

}
// --- end of MyPushbackReader.java

// --- file: WebScanner.java
//
import java.io.*;

class WebScanner
{
  MyPushbackReader in;
  String token;        // The current token
  int    ich;          // input character as int
  char   ch;           // current input character

  WebScanner ( Reader rdr )
  {
    in = new MyPushbackReader( rdr );
  }

  // is the character white space?
  private boolean whiteSpace( int ch )
  {
    if ( ch==' ' || ch=='\t' || ch=='\n' || ch=='\r' ) 
      return true;
    else
      return false;
  }

  // Scan thru the input stream character
  // by character.  Set token to the next token.
  // Return the token, or null on eof.
  String nextToken() throws IOException
  {
    // states of the automaton
    final int start=1;
    final int word =2;
    final int tag  =3;
    int       state;

    StringBuffer buff = new StringBuffer();
    state = start;

    while ( (ich = in.read()) != -1 )
    {

      ch = (char)ich;

      if ( state==start && whiteSpace(ch) )
        state = start;

      else if ( state==start && ch == '<' )
      {
        state = tag;
        buff.append( ch );
      }

      else if ( state==start )
      {
        state = word;
        buff.append( ch );
      }

      else if ( state==word && whiteSpace(ch) )
      {
        token = buff.toString().trim() ;
        return token;
      }

      else if ( state==word && ch == '<' )
      {
        in.unread( ch );
        token = buff.toString().trim() ;
        return token;
      }

      else if ( state==word )
      {
        buff.append( ch );
      }

      else if ( state==tag && ch == '>' )
      {
        buff.append( ch );
        token = buff.toString().trim() ;
        return token;
      }

      else if ( state==tag )
      {
        buff.append( ch );
      }
    
    } // end while

    return null;   // end of file

  } // end nextToken()

} // end WebScanner
// --- end of: WebScanner.java