/*
 *        JacORB - a free Java ORB
 *
 *   Copyright (C) 1997-2001  Gerald Brose.
 *
 *   This library is free software; you can redistribute it and/or
 *   modify it under the terms of the GNU Library General Public
 *   License as published by the Free Software Foundation; either
 *   version 2 of the License, or (at your option) any later version.
 *
 *   This library is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *   Library General Public License for more details.
 *
 *   You should have received a copy of the GNU Library General Public
 *   License along with this library; if not, write to the Free
 *   Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

package org.jacorb.idl;

import java.util.*;
import java_cup.runtime.token;
import java_cup.runtime.int_token;
import java_cup.runtime.char_token;
import java_cup.runtime.float_token;

/** 
 *  This  class implements  a small scanner  (aka lexical  analyzer or
 *  lexer) for  IDL. The scanner reads characters  from a global input
 *  stream  and returns integers corresponding to  the terminal number
 *  of the next token.  Once the end of input is reached the EOF token
 *  is returned on every  subsequent call.<p> 
 *
 *  All symbol constants are defined in sym.java which is generated by
 *  JavaCup from parser.cup.<p>
 * 
 *  In  addition to the scanner  proper (called first  via init() then
 *  with  next_token() to get  each token) this class  provides simple
 *  error  and warning  routines  and keeps  a  count  of errors  and
 *  warnings that  is  publicly accessible.  It  also provides  basic
 *  preprocessing   facilties,  i.e.  it   does  handle  preprocessor
 *  directives  such as #define,  #undef, #include, etc.   although it
 *  does not provide full C++ preprocessing
 *  
 *  This class is "static" (i.e., it has only static members and methods).
 *
 * @version $Id: lexer.java,v 1.10 2001/06/13 09:54:50 jacorb Exp $
 * @author Gerald Brose
 * 
 */

public class lexer 
{
    /** First and second character of lookahead. */
    protected static int next_char; 
    protected static int next_char2;

    /** EOF constant. */
    protected static final int EOF_CHAR = -1;

    /** 
     * Table   of  keywords.   Keywords  are   initially  treated  as
     * identifiers.  Just before they  are returned we look them up in
     * this table  to see  if they  match one  of the  keywords.  The
     * string of  the name  is the  key here,  which  indexes Integer
     * objects holding the symbol number.  
     */

    protected static Hashtable keywords = new Hashtable();

    /** Table of keywords, stored in lower case.  Keys are the
     *  lower case version of the keywords used as keys for the keywords
     *  hash above, and the values are the case sensitive versions of
     *  the keywords.  This table is used for detecting collisions of
     *  identifiers with keywords.
     */
    protected static Hashtable keywords_lower_case = new Hashtable();
    

    /** Table of Java reserved names. 
     */

    protected static Hashtable java_keywords = new Hashtable();

    /** Table of single character symbols.  For ease of implementation, we 
     *  store all unambiguous single character tokens in this table of Integer
     *  objects keyed by Integer objects with the numerical value of the 
     *  appropriate char (currently Character objects have a bug which precludes
     *  their use in tables).
     */
    protected static Hashtable char_symbols = new Hashtable(25);


    /** Defined symbols  (preprocessor) */

    protected static Hashtable defines = new Hashtable();
    protected static boolean conditionalCompilation = true;

    /** nested #ifdefs are pushed on this stack by the "preprocessor" */

    private static java.util.Stack ifStack = new Stack();
    private static java.util.Stack tokenStack = new Stack();

    /** Current line number for use in error messages. */
    protected static int current_line = 1;

    /** Current line  for use in error messages. */
    protected static StringBuffer line = new StringBuffer();

    /** Character position in current line. */
    protected static int current_position = 1;

    /** Have we already read a '"' ?  */
    protected static boolean in_string = false;

    /** Count of total errors detected so far. */
    public static int error_count = 0;

    /** Count of warnings issued so far */
    public static int warning_count = 0;

    /** currently active pragma prefix */
    public static String currentPragmaPrefix = "" ;


    /** reset the scanner state */

    public static void reset()
    {
        current_position = 1;
        error_count = 0;
        warning_count = 0;
        currentPragmaPrefix = "" ;
        line = new StringBuffer();
        ifStack.removeAllElements();
        tokenStack.removeAllElements();
        defines.clear();
   }

    /** 
     * Initialize the scanner.  This sets up the keywords and char_symbols
     * tables and reads the first two characters of lookahead. 
     *
     * "Object" is listed as reserved in the OMG spec. 
     * "int" is not, but I reserved it to bar its usage as a legal integer
     * type. 
     */

    public static void init() 
        throws java.io.IOException
    {
        /* set up standard symbols */
        defines.put("JACORB_IDL_1_3","");

        /* set up the keyword table */

        keywords.put("abstract",        new Integer(sym.ABSTRACT));
        keywords.put("any",     new Integer(sym.ANY));
        keywords.put("attribute",       new Integer(sym.ATTRIBUTE));
        keywords.put("boolean", new Integer(sym.BOOLEAN));
        keywords.put("case",    new Integer(sym.CASE));
        keywords.put("char",    new Integer(sym.CHAR));
        keywords.put("const",   new Integer(sym.CONST));
        keywords.put("context", new Integer(sym.CONTEXT));
        keywords.put("custom",  new Integer(sym.CUSTOM));
        keywords.put("default", new Integer(sym.DEFAULT));
        keywords.put("double",  new Integer(sym.DOUBLE));
        keywords.put("enum",    new Integer(sym.ENUM));
        keywords.put("exception",       new Integer(sym.EXCEPTION));
        keywords.put("factory", new Integer(sym.FACTORY));
        keywords.put("FALSE",   new Integer(sym.FALSE));
        keywords.put("fixed",   new Integer(sym.FIXED));
        keywords.put("float",   new Integer(sym.FLOAT));
        keywords.put("in",      new Integer(sym.IN));
        keywords.put("inout",   new Integer(sym.INOUT));
        keywords.put("interface",       new Integer(sym.INTERFACE));
        keywords.put("long",    new Integer(sym.LONG));
        keywords.put("module",  new Integer(sym.MODULE));
        keywords.put("native",  new Integer(sym.NATIVE));
        keywords.put("Object",  new Integer(sym.OBJECT));
        keywords.put("octet",   new Integer(sym.OCTET));
        keywords.put("oneway",  new Integer(sym.ONEWAY));
        keywords.put("out",     new Integer(sym.OUT));
        keywords.put("private", new Integer(sym.PRIVATE));
        keywords.put("public",  new Integer(sym.PUBLIC));
        keywords.put("pseudo",  new Integer(sym.PSEUDO));
        keywords.put("raises",  new Integer(sym.RAISES));
        keywords.put("readonly",        new Integer(sym.READONLY));
        keywords.put("sequence",        new Integer(sym.SEQUENCE));
        keywords.put("short",   new Integer(sym.SHORT));
        keywords.put("string",  new Integer(sym.STRING));
        keywords.put("struct",  new Integer(sym.STRUCT));
        keywords.put("supports",        new Integer(sym.SUPPORTS));
        keywords.put("switch",  new Integer(sym.SWITCH));
        keywords.put("TRUE",    new Integer(sym.TRUE));
        keywords.put("truncatable",     new Integer(sym.TRUNCATABLE));
        keywords.put("typedef", new Integer(sym.TYPEDEF));
        keywords.put("unsigned",        new Integer(sym.UNSIGNED));
        keywords.put("union",   new Integer(sym.UNION));
        keywords.put("ValueBase",       new Integer(sym.VALUEBASE));
        keywords.put("valuetype",       new Integer(sym.VALUETYPE));
        keywords.put("void",    new Integer(sym.VOID));
        keywords.put("wchar",   new Integer(sym.WCHAR));
        keywords.put("wstring", new Integer(sym.WSTRING));
  
        keywords.put("int",     new Integer(sym.INT));

        keywords.put("::", new Integer(sym.DBLCOLON));
        keywords.put("<<", new Integer(sym.LSHIFT));
        keywords.put(">>", new Integer(sym.RSHIFT));

        // setup the mapping of lower case keywords to case sensitive
        // keywords

        for ( java.util.Enumeration e = keywords.keys(); e.hasMoreElements(); )
        {
            String keyword = (String)e.nextElement();
            String keyword_lower_case = keyword.toLowerCase();
            keywords_lower_case.put ( keyword_lower_case, keyword );
        }

        /* set up the table of single character symbols */
        char_symbols.put(new Integer(';'), new Integer(sym.SEMI));
        char_symbols.put(new Integer(','), new Integer(sym.COMMA));
        char_symbols.put(new Integer('*'), new Integer(sym.STAR));
        char_symbols.put(new Integer('.'), new Integer(sym.DOT));
        char_symbols.put(new Integer(':'), new Integer(sym.COLON));
        char_symbols.put(new Integer('='), new Integer(sym.EQUALS));
        char_symbols.put(new Integer('+'), new Integer(sym.PLUS));
        char_symbols.put(new Integer('-'), new Integer(sym.MINUS));
        char_symbols.put(new Integer('{'), new Integer(sym.LCBRACE));
        char_symbols.put(new Integer('}'), new Integer(sym.RCBRACE));
        char_symbols.put(new Integer('('), new Integer(sym.LPAREN));
        char_symbols.put(new Integer(')'), new Integer(sym.RPAREN));
        char_symbols.put(new Integer('['), new Integer(sym.LSBRACE));
        char_symbols.put(new Integer(']'), new Integer(sym.RSBRACE));
        char_symbols.put(new Integer('<'), new Integer(sym.LESSTHAN));
        char_symbols.put(new Integer('>'), new Integer(sym.GREATERTHAN));
        char_symbols.put(new Integer('\''), new Integer(sym.QUOTE));
        char_symbols.put(new Integer('\"'), new Integer(sym.DBLQUOTE));
        char_symbols.put(new Integer('\\'), new Integer(sym.BSLASH));
        char_symbols.put(new Integer('^'), new Integer(sym.CIRCUM));
        char_symbols.put(new Integer('&'), new Integer(sym.AMPERSAND));
        char_symbols.put(new Integer('/'), new Integer(sym.SLASH));
        char_symbols.put(new Integer('%'), new Integer(sym.PERCENT));
        char_symbols.put(new Integer('~'), new Integer(sym.TILDE));
        char_symbols.put(new Integer('|'), new Integer(sym.BAR));

        /* set up reserved Java names */

        java_keywords.put("abstract",   "");
        java_keywords.put("boolean",    "");
        java_keywords.put("break",      "");
        java_keywords.put("byte",       "");
        java_keywords.put("case",       "");
        java_keywords.put("catch",      "");
        java_keywords.put("char",       "");
        java_keywords.put("class",      "");
        java_keywords.put("const",      "");
        java_keywords.put("continue",   "");
        java_keywords.put("default",    "");
        java_keywords.put("do", "");
        java_keywords.put("double",     "");
        java_keywords.put("else",       "");
        java_keywords.put("extends",    "");
        java_keywords.put("false",      "");    
        java_keywords.put("final",      "");
        java_keywords.put("finally",    "");
        java_keywords.put("for",        "");
        java_keywords.put("float",      "");
        java_keywords.put("goto",       "");
        java_keywords.put("if", "");
        java_keywords.put("implements", "");
        java_keywords.put("import",     "");
        java_keywords.put("instanceof", "");
        java_keywords.put("int",        "");
        java_keywords.put("interface",  "");
        java_keywords.put("long",       "");
        java_keywords.put("native",     "");
        java_keywords.put("new",        "");
        java_keywords.put("null",       "");
        java_keywords.put("package",    "");
        java_keywords.put("protected",  "");
        java_keywords.put("private",    "");
        java_keywords.put("public",     "");
        java_keywords.put("return",     "");
        java_keywords.put("short",      "");
        java_keywords.put("static",     "");
        java_keywords.put("super",      "");
        java_keywords.put("switch",     "");
        java_keywords.put("synchronized",       "");
        java_keywords.put("true",       "");
        java_keywords.put("this",       "");
        java_keywords.put("throw",      "");
        java_keywords.put("throws",     "");
        java_keywords.put("transient",  "");
        java_keywords.put("try",        "");
        java_keywords.put("volatile",   "");
        java_keywords.put("void",       "");
        java_keywords.put("while",      "");

        java_keywords.put("clone",      "");
        java_keywords.put("equals",     "");
        java_keywords.put("finalize",   "");
        java_keywords.put("getClass",   "");
        java_keywords.put("hashCode",   "");
        java_keywords.put("notify",     "");
        java_keywords.put("toString",   "");
        java_keywords.put("wait",       "");



        /* stack needs a topmost value */
        ifStack.push( new Boolean(true));

        /* read two characters of lookahead */

        try
        {
            next_char = GlobalInputStream.read();
        }
        catch (Exception e)
        {
            org.jacorb.idl.parser.fatal_error( "Cannot read from file " + 
                                               GlobalInputStream.currentFile().getAbsolutePath() + 
                                               ", please check file name.", null);
        }

        if (next_char == EOF_CHAR) 
            next_char2 = EOF_CHAR;
        else
            next_char2 = GlobalInputStream.read();
    }


    public static void define(String symbol, String value)
    {
        Environment.output(4,"Defining: " + symbol + " as " + value );
        defines.put(symbol, value);
    }

    public static void undefine(String symbol)
    {
        Environment.output(4,"Un-defining: " + symbol);
        defines.remove(symbol);
    }

    public static String defined(String symbol)
    {
        return (String)defines.get(symbol);
    }

    /** record information about the last lexical scope so that it can be
        restored later */


    public static int currentLine()
    {
        return current_line;
    }

    /**
     * asks the lexer to return its own
     */

    public static PositionInfo getPosition()
    {
        return new PositionInfo( current_line, 
                                 current_position, 
                                 currentPragmaPrefix, 
                                 line.toString() );
    }

    public static void restorePosition(PositionInfo p)
    {
        current_line = p.line_no;
        currentPragmaPrefix = p.pragma_prefix;
        current_position = 0;
    }

    /** 
     * Advance the scanner one character in the input stream.  This moves
     * next_char2 to next_char and then reads a new next_char2.  
     */

    protected static void advance() 
        throws java.io.IOException
    {
        int old_char;

        old_char = next_char;
        next_char = next_char2;
        next_char2 = GlobalInputStream.read();

        line.append( (char)old_char );

        /* count this */

        current_position++;
        if (old_char == '\n')
        {
            current_line++;
            current_position = 1;
            line = new StringBuffer();
        }
        //System.out.println("advance. next_char is " + next_char + " (" + (char)next_char + ")");
    }


    /** 
     *  Emit an error message.  The message will be marked with both the 
     *  current line number and the position in the line.  Error messages
     *  are printed on standard error (System.err).
     * @param message the message to print.
     * @param p_info an optional PositionInfo object
     */

    public static void emit_error(String message)
    {
        System.err.println( GlobalInputStream.currentFile().getAbsolutePath() + 
                            ", line: "+current_line +
                            "(" +current_position + "): " + message);

        System.err.println("\t" + line.toString() );
        error_count++;
    }

    public static void emit_error(String message, str_token t)
    {
        if( t == null )
        {
            emit_error( message );
        } 
        else 
        {
            System.err.println( "Error in " + t.fileName + ", line:" + t.line_no + 
                                "(" + t.char_pos + "): " + message );
            System.err.println("\t" + t.line_val );
        }
        error_count++;
    }


    /** Emit a warning message.  The message will be marked with both the 
     *  current line number and the position in the line.  Messages are 
     *  printed on standard error (System.err).
     * @param message the message to print.
     */

    public static void emit_warn(String message)
    {
        System.err.println("Warning: " + message + " at " + 
                           current_line + "(" + current_position +
                           "): \"" + line.toString() + "\"" );
        warning_count++;
    }


    public static void emit_warn(String message, str_token t)
    {
        if( t == null )
        {
            emit_warn( message );
        } 
        else 
        {
            System.err.println( "Warning at " + t.fileName + ", line:" + t.line_no + "(" + 
                                t.char_pos + "): " + message );
            System.err.println("\t" + t.line_val );
        }
        error_count++;
    }


    /** 
     * Determine if a character is ok to start an id. 
     * @param ch the character in question.
     */
    protected static boolean id_start_char(int ch)
    {
        return (ch >= 'a' &&  ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||  (ch == '_');
    }


    /**
     *  Determine if a character is ok for the middle of an id.
     * @param ch the character in question. 
     */
    protected static boolean id_char(int ch)
    {
        return id_start_char(ch) ||  (ch == '_') || (ch >= '0' && ch <= '9');
    }


    /**
     * Try to look up a single character symbol, returns -1 for not found. 
     * @param ch the character in question.
     */

    protected static int find_single_char(int ch)
    {
        Integer result;

        result = (Integer)char_symbols.get(new Integer((char)ch));
        if (result == null) 
            return -1;
        else
            return result.intValue();
    }

    /** 
     * Handle swallowing up a comment.  Both old style C and new style C++
     * comments are handled.
     */
    protected static void swallow_comment() 
        throws java.io.IOException
    {
        /* next_char == '/' at this point */

        /* is it a traditional comment */
        if (next_char2 == '*')
        {
                                /* swallow the opener */
            advance(); advance();

                                /* swallow the comment until end of comment or EOF */
            for (;;)
            {
                /* if its EOF we have an error */
                if (next_char == EOF_CHAR)
                {
                    emit_error("Specification file ends inside a comment", null);
                    return;
                }

                /* if we can see the closer we are done */
                if (next_char == '*' && next_char2 == '/')
                {
                    advance(); advance();
                    return;
                }

                /* otherwise swallow char and move on */
                advance();
            }
        }

        /* is its a new style comment */
        if (next_char2 == '/')
        {
                                /* swallow the opener */
            advance(); advance();

                                /* swallow to '\n', '\f', or EOF */ 
            while (next_char != '\n' && next_char != '\f' && next_char != '\r' && next_char!=EOF_CHAR)
            {
                advance();
            }

            return;
        }

        /* shouldn't get here, but... if we get here we have an error */
        emit_error("Malformed comment in specification -- ignored", null);
        advance();
    }

    /** 
     *  Preprocessor directives are handled here.
     */

    protected static void preprocess() 
        throws java.io.IOException
    {
        for (;;)
        {
            /* if its EOF we have an error */
            if (next_char == EOF_CHAR)
            {
                emit_error("Specification file ends inside a preprocessor directive", null);
                return;
            } 
            else if( next_char != '#' )
            {
                emit_error("expected #, got " + (char)next_char + " instead!", null);
            }
            else
                advance();      // skip '#'
                        
            // the following is done to allow for #   ifdef sloppiness
            while (  ( ' ' == next_char ) || ( '\t' == next_char ) )
                advance();        

            String dir = get_string();

            if( dir.equals("include"))
            {
                if( !conditionalCompilation)
                    return; 
                advance();      // skip ' '
                boolean useIncludePath = ( next_char == '<' );
                advance(); // skip `\"' or '<'
                String fname = get_string();
                if( useIncludePath && ( next_char != '>' ))
                    emit_error("Syntax error in #include directive, expecting '>'");
                else if( !useIncludePath && ( next_char != '\"' ))
                    emit_error("Syntax error in #include directive, expecting \"");
                                        
                /* swallow to '\n', '\f', or EOF */ 
                while (next_char != '\n' && next_char != '\f' && next_char != '\r' &&
                       next_char!=EOF_CHAR  )
                {
                    advance();
                }              
                GlobalInputStream.include( fname, (char)next_char2, useIncludePath );
                current_line = 0;
                advance();
                advance();
                //    System.out.println("returning from include, next_char is: " + (char)next_char);
                return;
            }
            else if( dir.equals("define"))
            {
                if( !conditionalCompilation)
                    return; 
                                //              advance();      // skip ' '
                swallow_whitespace();
                String name = get_string();
                StringBuffer text = new StringBuffer();
                if( next_char == ' ' )
                {
                    advance();
                }
                while( next_char != '\n' )
                {
                    if( next_char == '\\')
                    {
                        advance();advance();
                    }
                    text.append( (char)next_char );
                    advance();
                }
                // System.out.println("#Defined symbol " + name + " as: " + text.toString() );
                define( name, text.toString() );
            }
            else if( dir.equals("error"))
            {
                if( !conditionalCompilation)
                    return; 
                advance();      // skip ' '
                String name = get_string();
                emit_error( name );
            }
            else if( dir.equals("undef"))
            {
                if( !conditionalCompilation)
                    return ;
                swallow_whitespace(); // advance();     // skip ' '
                String name = get_string();
                undefine(name);
                // System.out.println("#Undefined symbol " + name  );
            }
            else if( dir.equals("if"))
            {
                ifStack.push( new Boolean(conditionalCompilation));
                if( !conditionalCompilation)
                    return ;
            
                swallow_whitespace();

                // the following snippet distinguishes between #if defined
                // and #if !defined
            
                boolean straightDefined = true;
                if ( '!' == next_char )
                {
                    advance();
                    straightDefined = false;
                }
                
                String defineStr = get_string_no_paren();
                if ( ! ( defineStr.equals("defined") ) )
                {
                    emit_error("Expected \"defined\" following #if: " + 
                               dir, null);
                    return;
                }

                swallow_whitespace();

                boolean brackets = ( '(' == next_char );
                if( brackets )
                {
                  advance(); // skip '('
                  swallow_whitespace(); // any whitespace after ( ? skip it
                }

                String name = get_string_no_paren();
            
                if( brackets )
                {
                    swallow_whitespace();
                    Environment.output(4, "next char: " + next_char );

                    if(  ')' != next_char )
                    {
                      emit_error("Expected ) terminating #if defined", null);
                    return;
                  }
                  advance();
                }

                if ( straightDefined )
                    conditionalCompilation = (null != defined(name));
                else
                    conditionalCompilation = (null == defined(name));
            }
            else if( dir.equals("ifdef"))
            {
                ifStack.push( new Boolean(conditionalCompilation));
                if( !conditionalCompilation)
                    return ;
                swallow_whitespace(); //advance();      // skip ' '
                String name = get_string();
                conditionalCompilation = ( defined(name) != null );
            }
            else if( dir.equals("ifndef"))
            {
                ifStack.push( new Boolean(conditionalCompilation));
                if( !conditionalCompilation)
                    return ;
                swallow_whitespace();//advance();       // skip ' '
                String name = get_string();
                conditionalCompilation = ( defined(name) == null );
            }
            else if( dir.equals("else"))
            {
                if( ((Boolean)ifStack.peek()).booleanValue())
                    conditionalCompilation = !conditionalCompilation;
            }
            else if( dir.equals("endif"))
            {
                boolean b = ((Boolean)ifStack.pop()).booleanValue();
                conditionalCompilation = b;
            }
            else if( dir.equals("pragma"))
            {
                if( !conditionalCompilation)
                    return ;
                advance();      // skip ' '
                String name = get_string();
                if( name.equals("prefix"))
                {
                    advance(); // skip ' '
                    //              if( (char)next_char != '\"' )
                    //                          emit_error("Expecting \" ");
                    //                      advance(); // skip '"'
                    String prefix = get_string();
                    //                      if( (char)next_char != '\"' )
                    //                          emit_error("Expecting \" ");
                    //                      advance(); // skip '"'
                    currentPragmaPrefix = prefix;                                       
                }
                else if( name.equals("version"))
                {
                    advance(); // skip ' '
                    String vname = get_string();
                    advance(); // skip ' '
                    String version = get_string();
                    parser.currentScopeData().versionMap.put(vname,version);
                } 
                else if( name.equals("ID"))
                {
                    advance(); // skip ' '
                    String iname = get_string();
                    advance(); // skip ' '
                    String id = get_string();
                    // do something with it
                }
                else if( name.equals("local"))
                {
                    /* proprietary pragma of the JacORB IDL compiler */
                    parser.setLocalityContraint();
                }
                else if( name.equals("inhibit_code_generation"))
                {
                    /* proprietary pragma of the JacORB IDL compiler */
                    parser.setInhibitionState(true);
                    // do something with it
                }
                else
                    emit_warn("Unknown pragma, ignoring: #pragma " + name, null);
            }
            else
                emit_error("Unrecognized preprocessor directive " + dir, null);


                                /* swallow to '\n', '\f', or EOF */ 
            while (next_char != '\n' && next_char != '\f' && next_char != '\r' &&
                   next_char!=EOF_CHAR  )
            {
                // System.out.println("Advancing after directive, next_char is: " + (char)next_char);
                advance();
            }
            return;
        }
    }


    // the following is used for parsing the #if defined(...) construct
    
    private static String get_string_no_paren()
        throws java.io.IOException
    {
        StringBuffer sb = new StringBuffer();
        char c = (char)next_char;
        while( c != ' ' && c != '\t' && c != '\r' && c != '\n' && c != '\f' &&  c != EOF_CHAR 
               && c != '\"' && c != '<' && c != '>' && c != '(' && c != ')' )
        {
            sb.append( c );
            advance();
            c = (char)next_char;
        }
        //      System.out.println("get string returns " + sb.toString());
        return sb.toString();
    }



    private static String get_string()
        throws java.io.IOException
    {

        StringBuffer sb = new StringBuffer("");

        if( next_char == '\"' )
        {
            advance();
            while( next_char != '\"' )
            {
                if( next_char == EOF_CHAR )
                    emit_error("Unexpected EOF in string");
                sb.append( (char)next_char );
                advance();
            }
        }
        else
        {
            while( next_char != ' ' && next_char != '\t' && next_char != '\r' && 
                   next_char != '\n' && next_char != '\f' &&  next_char != EOF_CHAR &&
                   next_char != '\"' && next_char != '<' && next_char != '>' )
            {
                sb.append( (char)next_char );
                advance();
            }
        }
        return sb.toString();
    }


    /** 
     *  Process an identifier.  Identifiers begin with a letter, underscore,
     *  or dollar sign, which is followed by zero or more letters, numbers,
     *  underscores or dollar signs.  This routine returns a str_token suitable
     *  for return by the scanner or null, if the string that was read expanded to
     *  a symbol that was #defined.In this case, the symbol is expanded in place
     */

    protected static token do_symbol() 
        throws java.io.IOException
    {
        StringBuffer result = new StringBuffer();
        String       result_str;
        Integer      keyword_num = null;
        char         buffer[] = new char[1];

        /* next_char holds first character of id */
        buffer[0] = (char)next_char;
        result.append(buffer,0,1);
        advance();

        /* collect up characters while they fit in id */ 
        while(id_char(next_char))
        {
            buffer[0] = (char)next_char;
            result.append(buffer,0,1);
            advance();
        }

        /* extract a string */
        result_str = result.toString();

        /* try to look it up as a defined symbol... */

        String text = defined( result_str );
        if( text != null )
        {
            char [] next = {(char)next_char,(char)next_char2};
            GlobalInputStream.insert(text + ( new String(next)) );
                                //System.out.println("Advancing after symbol " + result_str );
            advance(); // restore lookahead
            advance(); // restore lookahead
            return null;
        }

        // check if it's a keyword
    
        keyword_num = (Integer)keywords.get(result_str);
        if ( keyword_num != null )
        {
            if( isScope( result_str ))
            {
                parser.openScope();
            }
            return new token(keyword_num.intValue());
        }
    
        // not a keyword, so treat as identifier after verifying
        // case sensitivity rules and prefacing with an _
        // if it collides with a Java keyword.
    
        result_str = checkIdentifier ( result_str );
        if ( null != result_str )
            return new str_token(sym.ID, result_str, getPosition(), 
                                 GlobalInputStream.currentFile().getName() );
        else
            return null;
    }

    private static boolean isScope( String keyword )
    {
        return ( keyword.equals("module") || 
                 keyword.equals("interface") ||
                 keyword.equals("struct") ||
                 keyword.equals("exception") ||
                 keyword.equals("union") 
                 //                 keyword.equals("valuetype") 
                 );
    }


    /**
     * checks whether Identifier str is legal and returns it,
     * prepends an underscore if necessary to avoid clashes with 
     * Java reserved words
     */

    public static String checkIdentifier( String str )
    {
        //Environment.output(4, "checking identifier " + str );

        /* if it is not an escaped identifier, look it up as a keyword */
        if( str.charAt(0) ==  '_' )
        {
            return str.substring(1);
        }

        String colliding_keyword = 
            (String)keywords_lower_case.get(str.toLowerCase());

        if ( colliding_keyword != null )
        {
            emit_error("Identifier " + str + " collides with keyword " + 
                       colliding_keyword + ".");
            return null;
        }

        if( needsJavaEscape( str ))
        {
            // Environment.output(4, "checking identifier " + str + " : needs _ ");
            return "_" + str;
        }

        // Environment.output(4, "checking identifier " + str + " : needs no _ ");       
        return str;
    }

    /**
     * Only the most general name clashes with Java keywords
     * are caught here. Identifiers need to be checked again 
     * at different other places in the compiler!
     */

    private static boolean needsJavaEscape( String s )
    {
        return( java_keywords.containsKey(s) );
    }

    public static boolean strictJavaEscapeCheck( String s )
    {        
        return( (!s.equals("Helper") && s.endsWith("Helper")) || 
                (!s.equals("Holder") && s.endsWith("Holder")) || 
                (!s.equals("Operations") && s.endsWith("Operations")) || 
                (!s.equals("POA") && s.endsWith("POA")) || 
                (!s.equals("POATie") && s.endsWith("POATie")));
    }

    public static boolean needsJavaEscape(Module m)
    {
        String s = m.pack_name;
        return ( !s.equals("Package") && s.endsWith("Package") ||
                 strictJavaEscapeCheck(s)
                );
    }


    /** 
     *  Return one token.  This is the main external interface to the scanner.
     *  It consumes sufficient characters to determine the next input token
     *  and returns it.
     */

    public static token next_token() 
        throws java.io.IOException
    {
        parser.set_included( GlobalInputStream.includeState());
        token result = real_next_token();
        // System.out.println("# next_token() => " + result.sym);
        return result;
    }

    private static void swallow_whitespace()
        throws java.io.IOException
    {
        /* look for white space */
        while (next_char == ' ' || next_char == '\t' || next_char == '\n' ||
               next_char == '\f' || next_char == '\r' )
        {
            /* advance past it and try the next character */
            //System.out.println("Swallowing whitespace: " + next_char +  (char)next_char);
            advance();
        }
    }

    /**
     *  The actual routine to return one token. 
     */

    protected static token real_next_token() 
        throws java.io.IOException
    {
        int sym_num;

        if( !tokenStack.empty())
            return (token)tokenStack.pop();

        for (;;)
        {
            if( !in_string )
            {    
                swallow_whitespace();

                /* look for preprocessor directives */
                if( (char)next_char == '#' )
                {
                    preprocess();
                    continue;                   
                }

                /* look for a comment */
                if (next_char == '/' && (next_char2 == '*' || next_char2 == '/'))
                {
                    /* swallow then continue the scan */
                    swallow_comment();
                    continue;
                }    
    
                if( !conditionalCompilation )
                {
                    advance(); 
                    if( next_char == EOF_CHAR )
                    {
                        emit_error("EOF in conditional compilation!", null);
                        return null;
                    }
                    else
                        continue;
                }

                /* look for COLON or DBLCOLON */
                if (next_char == ':')
                {
                    if (next_char2 == ':')
                    {
                        advance();
                        advance();
                        return new token(sym.DBLCOLON);
                    }
                    else
                    {
                        advance();
                        return new token(sym.COLON);
                    }
                }


                /* leading 0: */
                /* Try to scan octal/hexadecimal numbers, might even find a float */
    
                if (next_char == '0')
                {
                    int i_val = 0;
                    int radix = 8;
                    int digit = 0;
                    advance();
    
                    if( next_char == '.' )
                    {
                        StringBuffer f_string = new StringBuffer("0.");
                        advance();
        
                        while( next_char >= '0' && next_char <= '9')
                        {
                            f_string.append( (char)next_char );
                            advance();
                        }
        
                        float f_val = (new Float( f_string.toString() )).floatValue();
                        return new float_token(sym.FLOAT_NUMBER, f_val);
                    } 
                    else
                    {
                        if( next_char == 'x' || next_char == 'X' )
                        {
                            advance();
                            radix = 16;
                        }
                        digit = Character.digit( (char)next_char, radix );
                        while ( digit != -1 )
                        {
                            i_val = i_val * radix + digit;
                            advance();
                            digit = Character.digit( (char)next_char, radix );
                        }
                        return new int_token(sym.NUMBER, i_val);
                    }
                }
    
                /* Try to scan integer, floating point or fixed point literals */  
    
                if ( (next_char >= '0' && next_char <= '9') || next_char == '.' )
                {
                    StringBuffer value = new StringBuffer();
                    StringBuffer fraction = null;
                    int exp = 0;

                    /* read integer part */
                    while (next_char >= '0' && next_char <= '9')
                    {
                        value.append( (char)next_char);
                        // System.out.println("Read integer part " + value.toString());
                        advance();
                    }
    
                    /* read fraction */
                    if( next_char == '.' )
                    {
                        fraction = new StringBuffer();
                        advance();
            
                        // System.out.println("Reading fraction part");

                        while( next_char >= '0' && next_char <= '9')
                        {
                            fraction.append( (char)next_char );
                            advance();
                        }               
                    } 

                    if( next_char == 'e' || next_char == 'E' )
                    {                        
                        // System.out.println("Reading exponent");

                        if( fraction == null )
                            fraction = new StringBuffer();

                        fraction.append('e');
                        advance();
                        if( next_char == '-' || next_char == '+')
                        {
                            fraction.append( (char)next_char );
                            advance();    
                        }

                        while( next_char >= '0' && next_char <= '9')
                        {
                            fraction.append( (char)next_char );
                            advance();
                        }
                        
                        if( fraction.length() == 1 )
                        {
                            emit_error("Empty exponent in float/double.");
                            continue;
                        }

                        return new float_token( sym.FLOAT_NUMBER, 
                                                Float.valueOf( value.toString() + 
                                                               "." + 
                                                               fraction.toString()).floatValue());
                    }

                    if( next_char == 'd' || next_char == 'D' )
                    {
                        advance();
                        if( fraction == null )
                            fraction = new StringBuffer();

                        java.math.BigDecimal bi = 
                            new java.math.BigDecimal(value.toString() + "." + 
                                                     fraction.toString());
                        return new fixed_token(sym.FIXED_NUMBER, bi);

                    }

                    if( fraction == null )
                    {
                        /* integer */
                        return new int_token( sym.NUMBER, 
                                              Integer.parseInt( value.toString()));
                    }
                    else
                    {                   
                        try
                        {
                            float f = 
                                Float.valueOf( value.toString() + "." + 
                                               fraction.toString()).floatValue();
                            return new float_token(sym.FLOAT_NUMBER, f );
                        }
                        catch( NumberFormatException nf )
                        {
                            emit_error("Unexpected symbol:  " + 
                                       value.toString() + "." + 
                                       fraction.toString());
                        }
                    }
                }

                /* look for a single character symbol */
                sym_num = find_single_char(next_char);  

                if( (char)next_char == '\"' )
                {
                    in_string = true;
                    advance();
                    return new token(sym.DBLQUOTE);
                }
                
                if( (char)next_char == '\'' )
                {                   
                    advance();
                    token t = new char_token(sym.CH,(char)next_char);
                    advance();
                    if( (char)next_char == '\'' )
                    {               
                        tokenStack.push( new token(sym.QUOTE));
                        tokenStack.push(t);
                        advance();
                    }                    
                    else
                        emit_error("Expecting closing \'");
                    return new token(sym.QUOTE);
                }               
                

                if (sym_num != -1)
                {
                    /* found one -- advance past it and return a token for it */
                    advance();

                    return new token(sym_num);
                }

                /* look for an id or keyword */
                if (id_start_char(next_char))
                {
                    token t = do_symbol();
                    if( t != null )
                        return t;
                    else
                        continue;
                } 

                /* look for EOF */
                if (next_char == EOF_CHAR) 
                {
                    return new token(sym.EOF);
                }
            }
            else // in_string
            {
              
                /* empty string ? */
                if( (char)next_char == '\"' )
                {
                    in_string = false;
                    advance();
                    return new token(sym.DBLQUOTE);
                    //              return new org.jacorb.idl.str_token(sym.ID, "", getPosition());
                }           
                
                StringBuffer result = new StringBuffer();
        
                /* collect up characters while they fit in id */ 
                while( ((char)next_char) != '\"')
                {
                    result.append( (char)next_char );
                    advance();
                }

                // String s = checkIdentifier( result.toString() );
                String s = result.toString();

                /*  build and return an id token with an attached string */
                return new org.jacorb.idl.str_token(sym.ID, s , getPosition(), 
                                                GlobalInputStream.currentFile().getName());
            }

            /* if we get here, we have an unrecognized character */
            emit_warn("Unrecognized character '" + 
                      new Character((char)next_char) + "'(" +next_char+ ") -- ignored");

            /* advance past it */
            advance();
        }
    }

}




