$ template make_lexer()
$
$ // Generate the lexer.cpp file 
$ output $basedir + '/lexer.cpp'
/* This file was generated by SableCC (http://www.sablecc.org/). */

#include <sstream>

#include "node.h"
#include "token.h"
#include "lexer.h"

extern "C" {

static struct {
  const $namespace::_TypeInfo *type_info;
  const char *text;
} make_table[] = {
$ foreach {//token}
  {&$namespace::@ename::type_info, "${sablecc:escape_string_literal(string(@value))}"},
$ end foreach
};

$ foreach {lexer_data/accept_table/state}
static int _accept_${position()}[] = { [-foreach {i}-]${.}, [-end-]};
$ end foreach

static int *_accept[] = {
$ foreach {lexer_data/accept_table/state}
  _accept_${position()},
$ end
};

$ foreach {lexer_data/goto_table/state}
$   set i = {position()}
$   foreach {row}
$     set j = {position()}
$     foreach {goto}
$       set k = {position()}
static int _g_${$i}_${$j}_${$k}[] = {@low, @high, @state};
$     end foreach
$   end foreach
$ end foreach

$ foreach {lexer_data/goto_table/state}
$   set i = {position()}
$   foreach {row}
$     set j = {position()}
static int *_g_${$i}_${$j}[] = {
        (int*)${count(goto)},
$     foreach {goto}
$     set k = {position()}
        _g_${$i}_${$j}_${$k},
$     end foreach
};
$   end foreach
$ end foreach

$ foreach {lexer_data/goto_table/state}
$   set i = {position()}
static int **_g_${$i}[] = {
$   foreach {row}
$     set j = {position()}
        _g_${$i}_${$j},
$   end foreach
};
$ end foreach

static int ***_goto_table[] = {
$ foreach {lexer_data/goto_table/state}
$   set i = {position()}
        _g_${$i},
$ end
};

static $namespace::Lexer::State transitions[${count(tokens/token)}][${count(lexer_data/state)}] = {
$ foreach token in {//token}
  [- -]        {[- -]
$   foreach state in {/parser/lexer_data/state}
$     if {$token/transition[@from=$state/@name]}
        [- -]$namespace::Lexer::${$token/transition[@from=$state/@name]/@to}, [- -]
$     else
        [- -]$namespace::Lexer::@name, [- -]
$     end if
$   end foreach
},
$ end foreach
};

} // extern "C" {


$namespace::StreamReader::StreamReader ( const std::string& path)
{
  std::filebuf *fbuf = new std::filebuf();
  fbuf->open (path.c_str(), std::ios::in|std::ios::binary);
  inbuf = fbuf;
  do_delete = true;
  eof = false;
}

$namespace::Lexer::Lexer (StreamReader *stream_reader, bool own_reader)
     : stream_reader (stream_reader), own_reader (own_reader)
{
  state = ${lexer_data/state/@name};
  line = 0;
  pos = 0;
  cr = false;
}

$namespace::Lexer::Lexer (const std::string& path)
{
  stream_reader = new StreamReader (path);
  own_reader = true;
  state = ${lexer_data/state/@name};
  line = 0;
  pos = 0;
  cr = false;
}

$namespace::Lexer::~Lexer ()
{
  if ( own_reader ) delete stream_reader;
}

$namespace::Token $namespace::Lexer::peek ()
{
  while ( !token ) {
      token = getToken ();
      filter();
  }

  return token;
}

$namespace::Token $namespace::Lexer::next ()
{
  while ( !token ) {
      token = getToken ();
      filter();
  }

  Token ret = token;
  token = Token();
  return ret;
}

void $namespace::Lexer::filter ()
{
}

void $namespace::Lexer::unread (Token token)
{
  const std::string& text = token.getText();
  for ( int i = text.size() - 1; i >= 0; i-- )
      stream_reader->unget (text[i]);
  pos = token.getPos() - 1;
  line = token.getLine() - 1;
}

$namespace::Token $namespace::Lexer::getToken ()
{
  int dfa_state = 0;

  int start_pos = pos;
  int start_line = line;

  int accept_state = -1;
  int accept_token = -1;
  int accept_length = -1;
  int accept_pos = -1;
  int accept_line = -1;

  int ***goto_table = _goto_table[state];
  int *accept = _accept[state];
  text.clear();

  while (1) {
      int c = stream_reader->get();

      if ( c != -1 ) {
          switch (c) {
            case 10:
              if ( cr ) {
                  cr = false;
              } else {
                  line++;
                  pos = 0;
              }
              break;
            case 13:
              line++;
              pos = 0;
              cr = true;
              break;
            default:
              pos++;
              cr = false;
              break;

          }

          text.push_back(c);
          do {
              int oldState = (dfa_state < -1) ? (-2 - dfa_state) : dfa_state;

              dfa_state = -1;

              int** tmp1 =  goto_table[oldState];
              int low = 1;
              int high = (int)tmp1[0]; // hackish huh
//              int high = tmp1.length - 1;

              while(low <= high)
              {
                  int middle = (low + high) / 2;
                  int* tmp2 = tmp1[middle];

                  if(c < tmp2[0])
                  {
                      high = middle - 1;
                  }
                  else if(c > tmp2[1])
                  {
                      low = middle + 1;
                  }
                  else
                  {
                      dfa_state = tmp2[2];
                      break;
                  }
              }
          } while ( dfa_state < -1 );
      } else {
          dfa_state = -1;
      }

      if ( dfa_state >= 0 ) {
          if(accept[dfa_state] != -1) {
              accept_state = dfa_state;
              accept_token = accept[dfa_state];
              accept_length = text.size();
              accept_pos = pos;
              accept_line = line;
          }
      } else {
          if ( accept_state != -1 ) {
              // pushBack
              for ( int i = (int)text.size() - 1; i >= accept_length; i-- )
                  stream_reader->unget (text[i]);

              pos = accept_pos;
              line = accept_line;

              int rline = start_line + 1;
              int rpos = start_pos + 1;

              state = transitions[accept_token][state];

              if ( make_table[accept_token].type_info->token_is_mutable ) {
                  return _GenericNode::initToken (make_table[accept_token].type_info,
                      rline, rpos, std::string (text.begin(), text.begin() + accept_length));
              } else {
                  return _GenericNode::initToken (make_table[accept_token].type_info,
                      rline, rpos, make_table[accept_token].text);
              }
          } else {
              if( text.size() ) {
                  std::ostringstream str;
                  str << "[" << (start_line + 1) << "," << (start_pos + 1) <<
                      "] Unknown token: " << std::string(text.begin(), text.end());
                  throw LexerException (start_line + 1, start_pos + 1, str.str());
              } else {
                  return TEOF::make(start_line + 1, start_pos + 1);
              }
          }
      }
  }
}
$ end output

$ // Generate the lexer.h file 
$ output $incdir + '/lexer.h'
/* This file was generated by SableCC (http://www.sablecc.org/). */

#ifndef __${$namespace}__lexer_hh__
#define __${$namespace}__lexer_hh__

#include <iostream>
#include <fstream>
#include <streambuf>
#include <ios>

#include <stack>
#include <vector>

namespace $namespace {

class StreamReader {
public:
  inline StreamReader ($streambuf *inbuf, bool do_delete = false)
      : inbuf (inbuf), do_delete(do_delete), eof(false) { }

  StreamReader (const std::string& path);

  inline ~StreamReader () { if ( do_delete ) delete inbuf; }

  inline int get ()
  {
    if ( !stack.empty() ) {
        int ret = stack.top();
        stack.pop();
        return ret;
    } else if ( !eof ) {
        int n = inbuf->sbumpc();
        if ( n == -1 ) eof = true;
        return n;
    } else {
        return -1;
    }
  }

  inline int peek () const { return stack.empty() ? inbuf->sgetc() : stack.top(); }
  inline void unget (int value) { stack.push (value); }
  operator bool () { return !stack.empty() || peek() != $streambuf::traits_type::eof(); }
  bool is_open () { return *this; }

private:
  $streambuf *inbuf;
  std::stack<int> stack;
  bool do_delete;
  bool eof;
};

class Lexer {
public:
  /**
    * Create a new Lexer object reading input from stream_reader.
    * @@param stream_reader StreamReader object where from to read input
    * @@param own_reader whether we own the reader and delete on our destruction
    */
  Lexer (StreamReader* stream_reader, bool own_reader = false);
  Lexer (const std::string& path);
  virtual ~Lexer ();

  virtual Token peek ();
  virtual Token next ();

  enum State {
$ foreach {lexer_data/state}
    @name = @id,
$ end
  };

protected:
  Token getToken();
  virtual void filter ();
  void unread (Token token);

private:
  StreamReader* stream_reader;
  bool own_reader;

  int line;
  int pos;
  bool cr;

  Token token;
  State state;
  std::vector<int> text;
};

class LexerException : public Exception {
public:
  inline LexerException (int line, int pos, const std::string& msg) : Exception (msg), line(line), pos(pos) { }
  inline int getPos () const { return pos; }
  inline int getLine () const { return line; }

private:
  int line;
  int pos;
};

} // namespace $namespace {

#endif // !__${$namespace}__lexer_hh__
$ end output

$ end template