/*
 *        JacORB - a free Java ORB
 *
 *   Copyright (C) 1997-98  Gerald Brose.
 *
 *   This library is free software; you can redistribute it and/or
 *   modify it under the terms of the GNU Library General Public
 *   License as published by the Free Software Foundation; either
 *   version 2 of the License, or (at your option) any later version.
 *
 *   This library is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *   Library General Public License for more details.
 *
 *   You should have received a copy of the GNU Library General Public
 *   License along with this library; if not, write to the Free
 *   Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

package jacorb.idl;

/** 
 *  This class implements a small scanner (aka lexical analyzer or lexer) for
 *  IDL. It is adapted from one the Java_CUP lexer. 
 *  
 *  The scanner reads characters from a global input stream and returns 
 *  integers corresponding to the terminal number of the next token.  
 *  Once the end of input is reached the EOF token is returned on every 
 *  subsequent call.<p>
 *  All symbol constants are defined in sym.java which is generated by 
 *  JavaCup from parser.cup.<p>
 * 
 *  In addition to the scanner proper (called first via init() then with
 *  next_token() to get each token) this class provides simple error and 
 *  warning routines and keeps a count of errors and warnings that is 
 *  publicly accessible.<p>
 *
 *  Also, this class provides basic preprocessing facilties, i.e. it does
 *  handle preprocessor directives such as #define, #undef, #include, etc.
 *  although it does not provide full C++ preprocessing
 *  
 *  This class is "static" (i.e., it has only static members and methods).
 *
 * @version $Id: lexer.java,v 1.29 2000/10/28 13:24:25 brose Exp $
 * @author Gerald Brose
 *
 */

import java.util.*;
import java_cup.runtime.token;
import java_cup.runtime.int_token;
import java_cup.runtime.char_token;
import java_cup.runtime.float_token;

public class lexer 
{

    /** First and second character of lookahead. */
    protected static int next_char; 
    protected static int next_char2;

    /** EOF constant. */
    protected static final int EOF_CHAR = -1;

    /** Table of keywords.  Keywords are initially treated as identifiers.
     *  Just before they are returned we look them up in this table to see if
     *  they match one of the keywords.  The string of the name is the key here,
     *  which indexes Integer objects holding the symbol number. 
     */

    protected static Hashtable keywords = new Hashtable();

    /** Table of keywords, stored in lower case.  Keys are the
     *  lower case version of the keywords used as keys for the keywords
     *  hash above, and the values are the case sensitive versions of
     *  the keywords.  This table is used for detecting collisions of
     *  identifiers with keywords.
     */
    protected static Hashtable keywords_lower_case = new Hashtable();
    

    /** Table of Java reserved names. 
     */

    protected static Hashtable java_keywords = new Hashtable();

    /** Table of single character symbols.  For ease of implementation, we 
     *  store all unambiguous single character tokens in this table of Integer
     *  objects keyed by Integer objects with the numerical value of the 
     *  appropriate char (currently Character objects have a bug which precludes
     *  their use in tables).
     */
    protected static Hashtable char_symbols = new Hashtable(25);


    /** Defined symbols  (preprocessor) */

    protected static Hashtable defines = new Hashtable();
    protected static boolean conditionalCompilation = true;

    /** nested #ifdefs are pushed on this stack by the "preprocessor" */

    private static java.util.Stack ifStack = new Stack();

    private static java.util.Stack tokenStack = new Stack();

    /** Current line number for use in error messages. */
    protected static int current_line = 1;

    /** Current line  for use in error messages. */
    protected static StringBuffer line = new StringBuffer();

    /** Character position in current line. */
    protected static int current_position = 1;

    /** Have we already read a '"' ?. */
    protected static boolean in_string = false;

    /** Count of total errors detected so far. */
    public static int error_count = 0;

    /** Count of warnings issued so far */
    public static int warning_count = 0;

    /** currently active pragma prefix */
    public static String currentPragmaPrefix = "" ;

    /** 
     * Initialize the scanner.  This sets up the keywords and char_symbols
     * tables and reads the first two characters of lookahead. 
     *
     * "Object" is listed as reserved in the OMG spec. 
     * "int" is not, but I reserved it to bar its usage as a legal integer
     * type. 
     */

    public static void init() 
	throws java.io.IOException
    {
	/* set up standard symbols */
	defines.put("JACORB_IDL_1_2","");

	/* set up the keyword table */

	keywords.put("abstract",	new Integer(sym.ABSTRACT));
	keywords.put("any",	new Integer(sym.ANY));
	keywords.put("attribute",	new Integer(sym.ATTRIBUTE));
	keywords.put("boolean",	new Integer(sym.BOOLEAN));
	keywords.put("case",	new Integer(sym.CASE));
	keywords.put("char",	new Integer(sym.CHAR));
	keywords.put("const",	new Integer(sym.CONST));
	keywords.put("context",	new Integer(sym.CONTEXT));
	keywords.put("custom",	new Integer(sym.CUSTOM));
	keywords.put("default",	new Integer(sym.DEFAULT));
	keywords.put("double",	new Integer(sym.DOUBLE));
	keywords.put("enum",	new Integer(sym.ENUM));
	keywords.put("exception",	new Integer(sym.EXCEPTION));
	keywords.put("factory",	new Integer(sym.FACTORY));
	keywords.put("FALSE",	new Integer(sym.FALSE));
	keywords.put("fixed",	new Integer(sym.FIXED));
	keywords.put("float",	new Integer(sym.FLOAT));
	keywords.put("in",	new Integer(sym.IN));
	keywords.put("inout",	new Integer(sym.INOUT));
	keywords.put("interface",	new Integer(sym.INTERFACE));
	keywords.put("long",	new Integer(sym.LONG));
	keywords.put("module",	new Integer(sym.MODULE));
	keywords.put("native",	new Integer(sym.NATIVE));
	keywords.put("Object",	new Integer(sym.OBJECT));
	keywords.put("octet",	new Integer(sym.OCTET));
	keywords.put("oneway",	new Integer(sym.ONEWAY));
	keywords.put("out",	new Integer(sym.OUT));
	keywords.put("private",	new Integer(sym.PRIVATE));
	keywords.put("public",	new Integer(sym.PUBLIC));
       	keywords.put("pseudo",	new Integer(sym.PSEUDO));
	keywords.put("raises",	new Integer(sym.RAISES));
	keywords.put("readonly",	new Integer(sym.READONLY));
	keywords.put("sequence",	new Integer(sym.SEQUENCE));
	keywords.put("short",	new Integer(sym.SHORT));
	keywords.put("string",	new Integer(sym.STRING));
	keywords.put("struct",	new Integer(sym.STRUCT));
	keywords.put("supports",	new Integer(sym.SUPPORTS));
	keywords.put("switch",	new Integer(sym.SWITCH));
	keywords.put("TRUE",	new Integer(sym.TRUE));
	keywords.put("truncatable",	new Integer(sym.TRUNCATABLE));
	keywords.put("typedef",	new Integer(sym.TYPEDEF));
	keywords.put("unsigned",	new Integer(sym.UNSIGNED));
	keywords.put("union",	new Integer(sym.UNION));
	keywords.put("ValueBase",	new Integer(sym.VALUEBASE));
	keywords.put("valuetype",	new Integer(sym.VALUETYPE));
	keywords.put("void",	new Integer(sym.VOID));
	keywords.put("wchar",	new Integer(sym.WCHAR));
	keywords.put("wstring",	new Integer(sym.WSTRING));
  
	keywords.put("int",	new Integer(sym.INT));

	keywords.put("::", new Integer(sym.DBLCOLON));
	keywords.put("<<", new Integer(sym.LSHIFT));
	keywords.put(">>", new Integer(sym.RSHIFT));

	// setup the mapping of lower case keywords to case sensitive
	// keywords

	for ( java.util.Enumeration e = keywords.keys(); e.hasMoreElements(); )
	{
	    String keyword = (String)e.nextElement();
	    String keyword_lower_case = keyword.toLowerCase();
	    keywords_lower_case.put ( keyword_lower_case, keyword );
	}

	/* set up the table of single character symbols */
	char_symbols.put(new Integer(';'), new Integer(sym.SEMI));
	char_symbols.put(new Integer(','), new Integer(sym.COMMA));
	char_symbols.put(new Integer('*'), new Integer(sym.STAR));
	char_symbols.put(new Integer('.'), new Integer(sym.DOT));
	char_symbols.put(new Integer(':'), new Integer(sym.COLON));
	char_symbols.put(new Integer('='), new Integer(sym.EQUALS));
	char_symbols.put(new Integer('+'), new Integer(sym.PLUS));
	char_symbols.put(new Integer('-'), new Integer(sym.MINUS));
	char_symbols.put(new Integer('{'), new Integer(sym.LCBRACE));
	char_symbols.put(new Integer('}'), new Integer(sym.RCBRACE));
	char_symbols.put(new Integer('('), new Integer(sym.LPAREN));
	char_symbols.put(new Integer(')'), new Integer(sym.RPAREN));
	char_symbols.put(new Integer('['), new Integer(sym.LSBRACE));
	char_symbols.put(new Integer(']'), new Integer(sym.RSBRACE));
	char_symbols.put(new Integer('<'), new Integer(sym.LESSTHAN));
	char_symbols.put(new Integer('>'), new Integer(sym.GREATERTHAN));
	char_symbols.put(new Integer('\''), new Integer(sym.QUOTE));
	char_symbols.put(new Integer('\"'), new Integer(sym.DBLQUOTE));
	char_symbols.put(new Integer('\\'), new Integer(sym.BSLASH));
	char_symbols.put(new Integer('^'), new Integer(sym.CIRCUM));
	char_symbols.put(new Integer('&'), new Integer(sym.AMPERSAND));
	char_symbols.put(new Integer('/'), new Integer(sym.SLASH));
	char_symbols.put(new Integer('%'), new Integer(sym.PERCENT));
	char_symbols.put(new Integer('~'), new Integer(sym.TILDE));
	char_symbols.put(new Integer('|'), new Integer(sym.BAR));

	/* set up reserved Java names */

	java_keywords.put("abstract",	"");
	java_keywords.put("boolean",	"");
	java_keywords.put("break",	"");
	java_keywords.put("byte",	"");
	java_keywords.put("case",	"");
	java_keywords.put("catch",	"");
	java_keywords.put("char",	"");
	java_keywords.put("class",	"");
	java_keywords.put("const",	"");
	java_keywords.put("continue",	"");
	java_keywords.put("default",	"");
	java_keywords.put("do",	"");
	java_keywords.put("double",	"");
	java_keywords.put("else",	"");
	java_keywords.put("extends",	"");
	java_keywords.put("false",	"");	
	java_keywords.put("final",	"");
	java_keywords.put("finally",	"");
	java_keywords.put("for",	"");
	java_keywords.put("float",	"");
	java_keywords.put("goto",	"");
	java_keywords.put("if",	"");
	java_keywords.put("implements",	"");
	java_keywords.put("import",	"");
	java_keywords.put("instanceof",	"");
	java_keywords.put("int",	"");
	java_keywords.put("interface",	"");
	java_keywords.put("long",	"");
	java_keywords.put("native",	"");
	java_keywords.put("new",	"");
	java_keywords.put("null",	"");
	java_keywords.put("package",	"");
	java_keywords.put("protected",	"");
	java_keywords.put("private",	"");
	java_keywords.put("public",	"");
       	java_keywords.put("return",	"");
	java_keywords.put("short",	"");
	java_keywords.put("static",	"");
	java_keywords.put("super",	"");
	java_keywords.put("switch",	"");
	java_keywords.put("synchronized",	"");
	java_keywords.put("true",	"");
	java_keywords.put("this",	"");
	java_keywords.put("throw",	"");
	java_keywords.put("throws",	"");
	java_keywords.put("transient",	"");
	java_keywords.put("try",	"");
	java_keywords.put("volatile",	"");
	java_keywords.put("void",	"");
	java_keywords.put("while",	"");

	java_keywords.put("clone",	"");
	java_keywords.put("equals",	"");
	java_keywords.put("finalize",	"");
	java_keywords.put("getClass",	"");
	java_keywords.put("hashCode",	"");
	java_keywords.put("notify",	"");
	java_keywords.put("toString",	"");
	java_keywords.put("wait",	"");



	/* stack needs a topmost value */
	ifStack.push( new Boolean(true));

	/* read two characters of lookahead */

	try
	{
	    next_char = GlobalInputStream.read();
	}
	catch (Exception e)
	{
	    System.err.println("Cannot read from file " + 
			       GlobalInputStream.currentFile().getAbsolutePath() + 
			       ", please check file name.");
	    System.exit(1);
	}

	if (next_char == EOF_CHAR) 
	    next_char2 = EOF_CHAR;
	else
	    next_char2 = GlobalInputStream.read();
    }


    public static void define(String symbol, String value)
    {
	Environment.output(4,"Defining: " + symbol + " as " + value );
	defines.put(symbol, value);
    }

    public static void undefine(String symbol)
    {
	Environment.output(4,"Un-defining: " + symbol);
	defines.remove(symbol);
    }

    public static String defined(String symbol)
    {
	return (String)defines.get(symbol);
    }

    /** record information about the last lexical scope so that it can be
	restored later */


    public static int currentLine()
    {
	return current_line;
    }

    /**
     * asks the lexer to return its own
     */

    public static PositionInfo getPosition()
    {
	return new PositionInfo( current_line, 
				 current_position, 
				 currentPragmaPrefix, 
				 line.toString() );
    }

    public static void restorePosition(PositionInfo p)
    {
	current_line = p.line_no;
	currentPragmaPrefix = p.pragma_prefix;
	current_position = 0;
    }

    /** 
     * Advance the scanner one character in the input stream.  This moves
     * next_char2 to next_char and then reads a new next_char2.  
     */

    protected static void advance() 
	throws java.io.IOException
    {
	int old_char;

	old_char = next_char;
	next_char = next_char2;
	next_char2 = GlobalInputStream.read();

	line.append( (char)old_char );

	/* count this */

	current_position++;
	if (old_char == '\n')
	{
	    current_line++;
	    current_position = 1;
	    line = new StringBuffer();
	}
        //System.out.println("advance. next_char is " + next_char + " (" + (char)next_char + ")");
    }


    /** 
     *  Emit an error message.  The message will be marked with both the 
     *  current line number and the position in the line.  Error messages
     *  are printed on standard error (System.err).
     * @param message the message to print.
     * @param p_info an optional PositionInfo object
     */

    public static void emit_error(String message)
    {
	System.err.println( GlobalInputStream.currentFile().getAbsolutePath() + 
			    ", line: "+current_line +
			    "(" +current_position + "): " + message);

	System.err.println("\t" + line.toString() );
	error_count++;
    }

    public static void emit_error(String message, str_token t)
    {
	if( t == null )
	{
	    emit_error( message );
	} 
	else 
	{
	    System.err.println( "Error in " + t.fileName + ", line:" + t.line_no + 
				"(" + t.char_pos + "): " + message );
	    System.err.println("\t" + t.line_val );
	}
	error_count++;
    }


    /** Emit a warning message.  The message will be marked with both the 
     *  current line number and the position in the line.  Messages are 
     *  printed on standard error (System.err).
     * @param message the message to print.
     */

    public static void emit_warn(String message)
    {
	System.err.println("Warning: " + message + " at " + 
			   current_line + "(" + current_position +
			   "): \"" + line.toString() + "\"" );
	warning_count++;
    }


    public static void emit_warn(String message, str_token t)
    {
	if( t == null )
	{
	    emit_warn( message );
	} 
	else 
	{
	    System.err.println( "Warning at " + t.fileName + ", line:" + t.line_no + "(" + 
				t.char_pos + "): " + message );
	    System.err.println("\t" + t.line_val );
	}
	error_count++;
    }


    /** 
     * Determine if a character is ok to start an id. 
     * @param ch the character in question.
     */
    protected static boolean id_start_char(int ch)
    {
	return (ch >= 'a' &&  ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||  (ch == '_');
    }


    /**
     *  Determine if a character is ok for the middle of an id.
     * @param ch the character in question. 
     */
    protected static boolean id_char(int ch)
    {
	return id_start_char(ch) ||  (ch == '_') || (ch >= '0' && ch <= '9');
    }


    /**
     * Try to look up a single character symbol, returns -1 for not found. 
     * @param ch the character in question.
     */

    protected static int find_single_char(int ch)
    {
	Integer result;

	result = (Integer)char_symbols.get(new Integer((char)ch));
	if (result == null) 
	    return -1;
	else
	    return result.intValue();
    }

    /** 
     * Handle swallowing up a comment.  Both old style C and new style C++
     * comments are handled.
     */
    protected static void swallow_comment() 
	throws java.io.IOException
    {
	/* next_char == '/' at this point */

	/* is it a traditional comment */
	if (next_char2 == '*')
	{
				/* swallow the opener */
	    advance(); advance();

				/* swallow the comment until end of comment or EOF */
	    for (;;)
	    {
		/* if its EOF we have an error */
		if (next_char == EOF_CHAR)
		{
		    emit_error("Specification file ends inside a comment", null);
		    return;
		}

		/* if we can see the closer we are done */
		if (next_char == '*' && next_char2 == '/')
		{
		    advance(); advance();
		    return;
		}

		/* otherwise swallow char and move on */
		advance();
	    }
	}

	/* is its a new style comment */
	if (next_char2 == '/')
	{
				/* swallow the opener */
	    advance(); advance();

				/* swallow to '\n', '\f', or EOF */ 
	    while (next_char != '\n' && next_char != '\f' && next_char != '\r' && next_char!=EOF_CHAR)
	    {
		advance();
	    }

	    return;
	}

	/* shouldn't get here, but... if we get here we have an error */
	emit_error("Malformed comment in specification -- ignored", null);
	advance();
    }

    /** 
     *  Preprocessor directives are handled here.
     */

    protected static void preprocess() 
	throws java.io.IOException
    {
	for (;;)
	{
	    /* if its EOF we have an error */
	    if (next_char == EOF_CHAR)
	    {
		emit_error("Specification file ends inside a preprocessor directive", null);
		return;
	    } 
	    else if( next_char != '#' )
	    {
		emit_error("expected #, got " + (char)next_char + " instead!", null);
	    }
	    else
		advance();	// skip '#'
			
	    // the following is done to allow for #   ifdef sloppiness
	    while (  ( ' ' == next_char ) || ( '\t' == next_char ) )
		advance();        

	    String dir = get_string();

	    if( dir.equals("include"))
	    {
		if( !conditionalCompilation)
		    return; 
		advance();	// skip ' '
		boolean useIncludePath = ( next_char == '<' );
		advance(); // skip `\"' or '<'
		String fname = get_string();
		if( useIncludePath && ( next_char != '>' ))
		    emit_error("Syntax error in #include directive, expecting '>'");
		else if( !useIncludePath && ( next_char != '\"' ))
		    emit_error("Syntax error in #include directive, expecting \"");
					
		/* swallow to '\n', '\f', or EOF */ 
		while (next_char != '\n' && next_char != '\f' && next_char != '\r' &&
		       next_char!=EOF_CHAR  )
		{
		    advance();
		}	       
		GlobalInputStream.include( fname, (char)next_char2, useIncludePath );
		current_line = 0;
		advance();
		advance();
		//    System.out.println("returning from include, next_char is: " + (char)next_char);
		return;
	    }
	    else if( dir.equals("define"))
	    {
		if( !conditionalCompilation)
		    return; 
				//		advance();	// skip ' '
		swallow_whitespace();
		String name = get_string();
		StringBuffer text = new StringBuffer();
		if( next_char == ' ' )
		{
		    advance();
		}
		while( next_char != '\n' )
		{
		    if( next_char == '\\')
		    {
			advance();advance();
		    }
		    text.append( (char)next_char );
		    advance();
		}
		// System.out.println("#Defined symbol " + name + " as: " + text.toString() );
		define( name, text.toString() );
	    }
	    else if( dir.equals("error"))
	    {
		if( !conditionalCompilation)
		    return; 
		advance();	// skip ' '
		String name = get_string();
		emit_error( name );
	    }
	    else if( dir.equals("undef"))
	    {
		if( !conditionalCompilation)
		    return ;
		swallow_whitespace(); // advance();	// skip ' '
		String name = get_string();
		undefine(name);
		// System.out.println("#Undefined symbol " + name  );
	    }
	    else if( dir.equals("if"))
	    {
		ifStack.push( new Boolean(conditionalCompilation));
		if( !conditionalCompilation)
		    return ;
            
		swallow_whitespace();

		// the following snippet distinguishes between #if defined
		// and #if !defined
            
		boolean straightDefined = true;
		if ( '!' == next_char )
		{
		    advance();
		    straightDefined = false;
		}
                
		String defineStr = get_string_no_paren();
		if ( ! ( defineStr.equals("defined") ) )
		{
		    emit_error("Expected defined following #if: " + dir, null);
		    return;
		}

		swallow_whitespace();

		if ( '(' != next_char )
		{
		    emit_error("Expected ( following #if defined", null);
		    return;
		}

		advance();
		swallow_whitespace();

		String name = get_string_no_paren();
            
		swallow_whitespace();

		if ( ')' != next_char )
		{
		    emit_error("Expected ) terminating #if defined", null);
		    return;
		}

		advance();
            
		if ( straightDefined )
		    conditionalCompilation = (null != defined(name));
		else
		    conditionalCompilation = (null == defined(name));
	    }
	    else if( dir.equals("ifdef"))
	    {
		ifStack.push( new Boolean(conditionalCompilation));
		if( !conditionalCompilation)
		    return ;
		swallow_whitespace(); //advance();	// skip ' '
		String name = get_string();
		conditionalCompilation = ( defined(name) != null );
	    }
	    else if( dir.equals("ifndef"))
	    {
		ifStack.push( new Boolean(conditionalCompilation));
		if( !conditionalCompilation)
		    return ;
		swallow_whitespace();//advance();	// skip ' '
		String name = get_string();
		conditionalCompilation = ( defined(name) == null );
	    }
	    else if( dir.equals("else"))
	    {
		if( ((Boolean)ifStack.peek()).booleanValue())
		    conditionalCompilation = !conditionalCompilation;
	    }
	    else if( dir.equals("endif"))
	    {
		boolean b = ((Boolean)ifStack.pop()).booleanValue();
		conditionalCompilation = b;
	    }
	    else if( dir.equals("pragma"))
	    {
		if( !conditionalCompilation)
		    return ;
		advance();	// skip ' '
		String name = get_string();
		if( name.equals("prefix"))
		{
		    advance(); // skip ' '
		    //  	    if( (char)next_char != '\"' )
		    //  			emit_error("Expecting \" ");
		    //  		    advance(); // skip '"'
		    String prefix = get_string();
		    //  		    if( (char)next_char != '\"' )
		    //  			emit_error("Expecting \" ");
		    //  		    advance(); // skip '"'
		    currentPragmaPrefix = prefix;					
		}
		else if( name.equals("version"))
		{
		    advance(); // skip ' '
		    String vname = get_string();
		    advance(); // skip ' '
		    String version = get_string();
		    parser.currentScopeData().versionMap.put(vname,version);
		} 
		else if( name.equals("ID"))
		{
		    advance(); // skip ' '
		    String iname = get_string();
		    advance(); // skip ' '
		    String id = get_string();
		    // do something with it
		}
		else if( name.equals("local"))
		{
		    /* proprietary pragma of the JacORB IDL compiler */
		    parser.setLocalityContraint();
		}
		else if( name.equals("inhibit_code_generation"))
		{
		    /* proprietary pragma of the JacORB IDL compiler */
		    parser.setInhibitionState(true);
		    // do something with it
		}
		else
		    emit_warn("Unknown pragma, ignoring: #pragma " + name, null);
	    }
	    else
		emit_error("Unrecognized preprocessor directive " + dir, null);


				/* swallow to '\n', '\f', or EOF */ 
	    while (next_char != '\n' && next_char != '\f' && next_char != '\r' &&
		   next_char!=EOF_CHAR  )
	    {
		// System.out.println("Advancing after directive, next_char is: " + (char)next_char);
		advance();
	    }
	    return;
	}
    }


    // the following is used for parsing the #if defined(...) construct
    
    private static String get_string_no_paren()
	throws java.io.IOException
    {
	StringBuffer sb = new StringBuffer();
	char c = (char)next_char;
	while( c != ' ' && c != '\t' && c != '\r' && c != '\n' && c != '\f' &&  c != EOF_CHAR 
	       && c != '\"' && c != '<' && c != '>' && c != '(' && c != ')' )
	{
	    sb.append( c );
	    advance();
	    c = (char)next_char;
	}
	//	System.out.println("get string returns " + sb.toString());
	return sb.toString();
    }



    private static String get_string()
	throws java.io.IOException
    {

	StringBuffer sb = new StringBuffer("");

	if( next_char == '\"' )
	{
	    advance();
	    while( next_char != '\"' )
	    {
		if( next_char == EOF_CHAR )
		    emit_error("Unexpected EOF in string");
		sb.append( (char)next_char );
		advance();
	    }
	}
	else
	{
	    while( next_char != ' ' && next_char != '\t' && next_char != '\r' && 
		   next_char != '\n' && next_char != '\f' &&  next_char != EOF_CHAR &&
		   next_char != '\"' && next_char != '<' && next_char != '>' )
	    {
		sb.append( (char)next_char );
		advance();
	    }
	}
	return sb.toString();
    }


    /** 
     *  Process an identifier.  Identifiers begin with a letter, underscore,
     *  or dollar sign, which is followed by zero or more letters, numbers,
     *  underscores or dollar signs.  This routine returns a str_token suitable
     *  for return by the scanner or null, if the string that was read expanded to
     *  a symbol that was #defined.In this case, the symbol is expanded in place
     */

    protected static token do_symbol() 
	throws java.io.IOException
    {
	StringBuffer result = new StringBuffer();
	String       result_str;
	Integer      keyword_num = null;
	char         buffer[] = new char[1];

	/* next_char holds first character of id */
	buffer[0] = (char)next_char;
	result.append(buffer,0,1);
	advance();

	/* collect up characters while they fit in id */ 
	while(id_char(next_char))
	{
	    buffer[0] = (char)next_char;
	    result.append(buffer,0,1);
	    advance();
	}

	/* extract a string */
	result_str = result.toString();

	/* try to look it up as a defined symbol... */

	String text = defined( result_str );
	if( text != null )
	{
	    char [] next = {(char)next_char,(char)next_char2};
	    GlobalInputStream.insert(text + ( new String(next)) );
				//System.out.println("Advancing after symbol " + result_str );
	    advance(); // restore lookahead
	    advance(); // restore lookahead
	    return null;
	}

	// check if it's a keyword
    
	keyword_num = (Integer)keywords.get(result_str);
	if ( keyword_num != null )
        {
            if( isScope( result_str ))
            {
                parser.openScope();
            }
	    return new token(keyword_num.intValue());
        }
    
	// not a keyword, so treat as identifier after verifying
	// case sensitivity rules and prefacing with an _
	// if it collides with a Java keyword.
    
	result_str = checkIdentifier ( result_str );
	if ( null != result_str )
	    return new str_token(sym.ID, result_str, getPosition(), 
				 GlobalInputStream.currentFile().getName() );
	else
	    return null;
    }

    private static boolean isScope( String keyword )
    {
        return ( keyword.equals("module") || 
                 keyword.equals("interface") ||
                 keyword.equals("struct") ||
                 keyword.equals("exception") ||
                 keyword.equals("union") 
                 //                 keyword.equals("valuetype") 
                 );
    }


    /**
     * checks whether Identifier str is legal and returns it,
     * prepends an underscore if necessary to avoid clashes with 
     * Java reserved words
     */

    public static String checkIdentifier( String str )
    {
        Environment.output(4, "checking identifier " + str );


	/* if it is not an escaped identifier, look it up as a keyword */
	if( str.charAt(0) ==  '_' )
        {
	    return str.substring(1);
        }

	String colliding_keyword = (String)keywords_lower_case.get(str.toLowerCase());
	if ( colliding_keyword != null )
	{
	    emit_error("Identifier " + str + " collides with keyword " + colliding_keyword + ".");
	    return null;
	}

	if( needsJavaEscape( str ))
        {
            Environment.output(4, "checking identifier " + str + " : needs _ ");
	    return "_" + str;
        }

        Environment.output(4, "checking identifier " + str + " : needs no _ ");       
	return str;
    }

    /**
     * Only the most general name clasehs with Java keywords
     * are caught here. Identifiers need to be checked again 
     * at different other places in the compiler!
     */

    private static boolean needsJavaEscape(String s )
    {
	return( java_keywords.containsKey(s) );
    }

    public static boolean strictJavaEscapeCheck( String s )
    {        
	return( (!s.equals("Helper") && s.endsWith("Helper")) || 
		(!s.equals("Holder") && s.endsWith("Holder")) || 
		(!s.equals("Operations") && s.endsWith("Operations")) || 
		(!s.equals("POA") && s.endsWith("POA")) || 
		(!s.equals("POATie") && s.endsWith("POATie")));
    }

    public static boolean needsJavaEscape(Module m)
    {
	String s = m.pack_name;
	return ( !s.equals("Package") && s.endsWith("Package") ||
                 strictJavaEscapeCheck(s)
                );
    }


    /** 
     *  Return one token.  This is the main external interface to the scanner.
     *  It consumes sufficient characters to determine the next input token
     *  and returns it.
     */

    public static token next_token() 
	throws java.io.IOException
    {
	parser.set_included( GlobalInputStream.includeState());
	token result = real_next_token();
	// System.out.println("# next_token() => " + result.sym);
	return result;
    }

    private static void swallow_whitespace()
	throws java.io.IOException
    {
	/* look for white space */
	while (next_char == ' ' || next_char == '\t' || next_char == '\n' ||
	       next_char == '\f' || next_char == '\r' )
	{
				/* advance past it and try the next character */
				//System.out.println("Swallowing whitespace: " + next_char +  (char)next_char);
	    advance();
	}
    }

    /**
     *  The actual routine to return one token. 
     */
    protected static token real_next_token() 
	throws java.io.IOException
    {
	int sym_num;

	if( !tokenStack.empty())
	    return (token)tokenStack.pop();

	for (;;)
	{
	    if( !in_string )
	    {    
		swallow_whitespace();

		/* look for preprocessor directives */
		if( (char)next_char == '#' )
		{
		    preprocess();
		    continue;			
		}

		/* look for a comment */
		if (next_char == '/' && (next_char2 == '*' || next_char2 == '/'))
		{
		    /* swallow then continue the scan */
		    swallow_comment();
		    continue;
		}    
    
		if( !conditionalCompilation )
		{
		    advance(); 
		    if( next_char == EOF_CHAR )
		    {
			emit_error("EOF in conditional compilation!", null);
			return null;
		    }
		    else
			continue;
		}

		/* look for COLON or DBLCOLON */
		if (next_char == ':')
		{
		    if (next_char2 == ':')
		    {
			advance();
			advance();
			return new token(sym.DBLCOLON);
		    }
		    else
		    {
			advance();
			return new token(sym.COLON);
		    }
		}


		/* leading 0: */
		/* Try to scan octal/hexadecimal numbers, might even find a float */
    
		if (next_char == '0')
		{
		    int i_val = 0;
		    int radix = 8;
		    int digit = 0;
		    advance();
    
		    if( next_char == '.' )
		    {
			StringBuffer f_string = new StringBuffer("0.");
			advance();
	
			while( next_char >= '0' && next_char <= '9')
			{
			    f_string.append( (char)next_char );
			    advance();
			}
	
			float f_val = (new Float( f_string.toString() )).floatValue();
			return new float_token(sym.FLOAT_NUMBER, f_val);
		    } 
		    else
		    {
			if( next_char == 'x' || next_char == 'X' )
			{
			    advance();
			    radix = 16;
			}
			digit = Character.digit( (char)next_char, radix );
			while ( digit != -1 )
			{
			    i_val = i_val * radix + digit;
			    advance();
			    digit = Character.digit( (char)next_char, radix );
			}
			return new int_token(sym.NUMBER, i_val);
		    }
		}
    
		/* Try to scan integer, floating point or fixed point literals */  
    
		if ( (next_char >= '0' && next_char <= '9') || next_char == '.' )
		{
		    StringBuffer value = new StringBuffer();
		    StringBuffer fraction = null;
		    int exp = 0;

		    /* read integer part */
		    while (next_char >= '0' && next_char <= '9')
		    {
			value.append( (char)next_char);
			// System.out.println("Read integer part " + value.toString());
			advance();
		    }
    
		    /* read fraction */
		    if( next_char == '.' )
		    {
			fraction = new StringBuffer();
			advance();
	    
			// System.out.println("Reading fraction part");

			while( next_char >= '0' && next_char <= '9')
			{
			    fraction.append( (char)next_char );
			    advance();
			}	    	
		    } 

		    if( next_char == 'e' || next_char == 'E' )
		    {
			
			// System.out.println("Reading exponent");

			if( fraction == null )
			    fraction = new StringBuffer();

			fraction.append('e');
			advance();
			if( next_char == '-' )
			{
			    fraction.append( (char)next_char );
			    advance();    
			}

			while( next_char >= '0' && next_char <= '9')
			{
			    fraction.append( (char)next_char );
			    advance();
			}
			
			return new float_token(sym.FLOAT_NUMBER, 
					       Float.valueOf( value.toString() + "." + 
                                                              fraction.toString()).floatValue());

		    }

		    if( next_char == 'd' || next_char == 'D' )
		    {
			advance();
			if( fraction == null )
			    fraction = new StringBuffer();

			java.math.BigDecimal bi = 
                            new java.math.BigDecimal(value.toString() + "." + 
                                                     fraction.toString());
			return new fixed_token(sym.FIXED_NUMBER, bi);

		    }

		    if( fraction == null )
		    {
			/* integer */
			return new int_token(sym.NUMBER, 
					     Integer.parseInt( value.toString()));
		    }
		    else
		    {			
			return new float_token(sym.FLOAT_NUMBER, 
					       Float.valueOf( value.toString() + "." + 
                                                              fraction.toString()).floatValue());

		    }
		}

		/* look for a single character symbol */
		sym_num = find_single_char(next_char);	

		if( (char)next_char == '\"' )
		{
		    in_string = true;
		    advance();
		    return new token(sym.DBLQUOTE);
		}
		
		if( (char)next_char == '\'' )
		{		    
		    advance();
		    token t = new char_token(sym.CH,(char)next_char);
		    advance();
		    if( (char)next_char == '\'' )
		    {		    
			tokenStack.push( new token(sym.QUOTE));
			tokenStack.push(t);
			advance();
		    }
		    else
			emit_error("Expecting closing \'");
		    return new token(sym.QUOTE);
		}		
		

		if (sym_num != -1)
		{
		    /* found one -- advance past it and return a token for it */
		    advance();

		    return new token(sym_num);
		}

		/* look for an id or keyword */
		if (id_start_char(next_char))
		{
		    token t = do_symbol();
		    if( t != null )
			return t;
		    else
			continue;
		} 

		/* look for EOF */
		if (next_char == EOF_CHAR) 
		{
		    return new token(sym.EOF);
		}
	    }
	    else // in_string
	    {
	      
		/* empty string ? */
		if( (char)next_char == '\"' )
		{
		    in_string = false;
		    advance();
		    return new token(sym.DBLQUOTE);
		    //		    return new jacorb.idl.str_token(sym.ID, "", getPosition());
		}	    
		
		StringBuffer result = new StringBuffer();
	
		/* collect up characters while they fit in id */ 
		while( ((char)next_char) != '\"')
		{
		    result.append( (char)next_char );
		    advance();
		}

		String s = checkIdentifier( result.toString() );

		/* otherwise build and return an id token with an attached string */
		return new jacorb.idl.str_token(sym.ID, s , getPosition(), 
						GlobalInputStream.currentFile().getName());
	    }

				/* if we get here, we have an unrecognized character */
	    emit_warn("Unrecognized character '" + 
		      new Character((char)next_char) + "'(" +next_char+ ") -- ignored");

				/* advance past it */
	    advance();
	}
    }

}
