1  #include <antlr/config.hpp>
  2  
  3  #include <string>
  4  #include <list>
  5  #include <vector>
  6  #include <map>
  7  #include <utility>
  8  //#include <iostream>
  9  #include <iterator>
 10  #include <sstream>
 11  #include <cassert>
 12  
 13  #include <antlr/TokenStream.hpp>
 14  #include <antlr/TokenWithIndex.hpp>
 15  #include <antlr/BitSet.hpp>
 16  #include <antlr/TokenStreamRewriteEngine.hpp>
 17  
 18  #ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
 19  namespace antlr {
 20  #endif
 21  
 22  #ifndef NO_STATIC_CONSTS
 23  const size_t TokenStreamRewriteEngine::MIN_TOKEN_INDEX = 0;
 24  const int TokenStreamRewriteEngine::PROGRAM_INIT_SIZE = 100;
 25  #endif
 26  
 27  const char* TokenStreamRewriteEngine::DEFAULT_PROGRAM_NAME = "default";
 28  
 29  namespace {
 30  
 31  	struct compareOperationIndex {
 32  		typedef TokenStreamRewriteEngine::RewriteOperation RewriteOperation;
 33  		bool operator() ( const RewriteOperation* a, const RewriteOperation* b ) const
 34  		{
 35  			return a->getIndex() < b->getIndex();
 36  		}
 37  	};
 38  	struct dumpTokenWithIndex {
 39  		dumpTokenWithIndex( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
 40  		void operator() ( const RefTokenWithIndex& t ) {
 41  			out << "[txt='" << t->getText() << "' tp=" << t->getType() << " idx=" << t->getIndex() << "]\n";
 42  		}
 43  		ANTLR_USE_NAMESPACE(std)ostream& out;
 44  	};
 45  }
 46  
 47  TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream)
 48  : stream(upstream)
 49  , index(MIN_TOKEN_INDEX)
 50  , tokens()
 51  , programs()
 52  , discardMask()
 53  {
 54  }
 55  
 56  TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize )
 57  : stream(upstream)
 58  , index(MIN_TOKEN_INDEX)
 59  , tokens(initialSize)
 60  , programs()
 61  , discardMask()
 62  {
 63  }
 64  
 65  RefToken TokenStreamRewriteEngine::nextToken( void )
 66  {
 67  	RefTokenWithIndex t;
 68  	// suck tokens until end of stream or we find a non-discarded token
 69  	do {
 70  		t = RefTokenWithIndex(stream.nextToken());
 71  		if ( t )
 72  		{
 73  			t->setIndex(index);  // what is t's index in list?
 74  			if ( t->getType() != Token::EOF_TYPE ) {
 75  				tokens.push_back(t);  // track all tokens except EOF
 76  			}
 77  			index++;			// move to next position
 78  		}
 79  	} while ( t && discardMask.member(t->getType()) );
 80  	return RefToken(t);
 81  }
 82  
 83  void TokenStreamRewriteEngine::rollback( const std::string& programName,
 84  													  size_t instructionIndex )
 85  {
 86  	program_map::iterator rewrite = programs.find(programName);
 87  	if( rewrite != programs.end() )
 88  	{
 89  		operation_list& prog = rewrite->second;
 90  		operation_list::iterator
 91  			j = prog.begin(),
 92  			end = prog.end();
 93  
 94  		std::advance(j,instructionIndex);
 95  		if( j != end )
 96  			prog.erase(j, end);
 97  	}
 98  }
 99  
100  void TokenStreamRewriteEngine::originalToStream( std::ostream& out,
101  																 size_t start,
102  																 size_t end ) const
103  {
104  	token_list::const_iterator s = tokens.begin();
105  	std::advance( s, start );
106  	token_list::const_iterator e = s;
107  	std::advance( e, end-start );
108  	std::for_each( s, e, tokenToStream(out) );
109  }
110  
111  void TokenStreamRewriteEngine::toStream( std::ostream& out,
112  													  const std::string& programName,
113  													  size_t firstToken,
114  													  size_t lastToken ) const
115  {
116  	if( tokens.size() == 0 )
117  		return;
118  
119  	program_map::const_iterator rewriter = programs.find(programName);
120  
121  	if ( rewriter == programs.end() )
122  		return;
123  
124  	// get the prog and some iterators in it...
125  	const operation_list& prog = rewriter->second;
126  	operation_list::const_iterator
127  		rewriteOpIndex = prog.begin(),
128  		rewriteOpEnd = prog.end();
129  
130  	size_t tokenCursor = firstToken;
131  	// make sure we don't run out of the tokens we have...
132      if( lastToken > (tokens.size() - 1) ) {
133          lastToken = tokens.size() - 1;
134      }
135  
136      while ( tokenCursor <= lastToken )
137      {
138  //			std::cout << "tokenCursor = " << tokenCursor << " first prog index = " << (*rewriteOpIndex)->getIndex() << std::endl;
139  
140          if( rewriteOpIndex != rewriteOpEnd )
141          {
142              size_t up_to_here = std::min(lastToken,(*rewriteOpIndex)->getIndex());
143              while( tokenCursor < up_to_here )
144                  out << tokens[tokenCursor++]->getText();
145          }
146          while ( rewriteOpIndex != rewriteOpEnd &&
147                    tokenCursor == (*rewriteOpIndex)->getIndex() &&
148                    tokenCursor <= lastToken )
149          {
150              tokenCursor = (*rewriteOpIndex)->execute(out);
151              ++rewriteOpIndex;
152          }
153          if( tokenCursor <= lastToken )
154              out << tokens[tokenCursor++]->getText();
155      }
156  	// std::cout << "Handling tail operations # left = " << std::distance(rewriteOpIndex,rewriteOpEnd) << std::endl;
157  	// now see if there are operations (append) beyond last token index
158  	std::for_each( rewriteOpIndex, rewriteOpEnd, executeOperation(out) );
159  	rewriteOpIndex = rewriteOpEnd;
160  }
161  
162  void TokenStreamRewriteEngine::toDebugStream( std::ostream& out,
163  															 size_t start,
164  															 size_t end ) const
165  {
166  	token_list::const_iterator s = tokens.begin();
167  	std::advance( s, start );
168  	token_list::const_iterator e = s;
169  	std::advance( e, end-start );
170  	std::for_each( s, e, dumpTokenWithIndex(out) );
171  }
172  
173  void TokenStreamRewriteEngine::addToSortedRewriteList( const std::string& programName,
174  																		 RewriteOperation* op )
175  {
176  	program_map::iterator rewrites = programs.find(programName);
177  	// check if we got the program already..
178  	if ( rewrites == programs.end() )
179  	{
180  		// no prog make a new one...
181  		operation_list ops;
182  		ops.push_back(op);
183  		programs.insert(std::make_pair(programName,ops));
184  		return;
185  	}
186  	operation_list& prog = rewrites->second;
187  
188  	if( prog.empty() )
189  	{
190  		prog.push_back(op);
191  		return;
192  	}
193  
194  	operation_list::iterator i, end = prog.end();
195  	i = end;
196  	--i;
197  	// if at or beyond last op's index, just append
198  	if ( op->getIndex() >= (*i)->getIndex() ) {
199  		prog.push_back(op); // append to list of operations
200  		return;
201  	}
202  	i = prog.begin();
203  
204  	if( i != end )
205  	{
206  		operation_list::iterator pos = std::upper_bound( i, end, op, compareOperationIndex() );
207  		prog.insert(pos,op);
208  	}
209  	else
210  		prog.push_back(op);
211  }
212  
213  #ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
214  }
215  #endif