src/google/protobuf/io/tokenizer.h - third_party/protobuf - Git at Google

 // Protocol Buffers - Google's data interchange format
 // Copyright 2008 Google Inc.  All rights reserved.
 // https://developers.google.com/protocol-buffers/
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 // notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 // copyright notice, this list of conditions and the following disclaimer
 // in the documentation and/or other materials provided with the
 // distribution.
 //     * Neither the name of Google Inc. nor the names of its
 // contributors may be used to endorse or promote products derived from
 // this software without specific prior written permission.
 //
 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 // Author: kenton@google.com (Kenton Varda)
 //  Based on original Protocol Buffers design by
 //  Sanjay Ghemawat, Jeff Dean, and others.
 //
 // Class for parsing tokenized text from a ZeroCopyInputStream.

 #ifndef GOOGLE_PROTOBUF_IO_TOKENIZER_H__
 #define GOOGLE_PROTOBUF_IO_TOKENIZER_H__

 #include <string>
 #include <vector>
 #include <google/protobuf/stubs/common.h>

 namespace google {
 namespace protobuf {
 namespace io {

 class ZeroCopyInputStream;     // zero_copy_stream.h

 // Defined in this file.
 class ErrorCollector;
 class Tokenizer;

 // Abstract interface for an object which collects the errors that occur
 // during parsing.  A typical implementation might simply print the errors
 // to stdout.
 class LIBPROTOBUF_EXPORT ErrorCollector {
  public:
   inline ErrorCollector() {}
   virtual ~ErrorCollector();

   // Indicates that there was an error in the input at the given line and
   // column numbers.  The numbers are zero-based, so you may want to add
   // 1 to each before printing them.
   virtual void AddError(int line, int column, const string& message) = 0;

   // Indicates that there was a warning in the input at the given line and
   // column numbers.  The numbers are zero-based, so you may want to add
   // 1 to each before printing them.
   virtual void AddWarning(int /* line */, int /* column */,
                           const string& /* message */) { }

  private:
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ErrorCollector);
 };

 // This class converts a stream of raw text into a stream of tokens for
 // the protocol definition parser to parse.  The tokens recognized are
 // similar to those that make up the C language; see the TokenType enum for
 // precise descriptions.  Whitespace and comments are skipped.  By default,
 // C- and C++-style comments are recognized, but other styles can be used by
 // calling set_comment_style().
 class LIBPROTOBUF_EXPORT Tokenizer {
  public:
   // Construct a Tokenizer that reads and tokenizes text from the given
   // input stream and writes errors to the given error_collector.
   // The caller keeps ownership of input and error_collector.
   Tokenizer(ZeroCopyInputStream* input, ErrorCollector* error_collector);
   ~Tokenizer();

   enum TokenType {
     TYPE_START,       // Next() has not yet been called.
     TYPE_END,         // End of input reached.  "text" is empty.

     TYPE_IDENTIFIER,  // A sequence of letters, digits, and underscores, not
                       // starting with a digit.  It is an error for a number
                       // to be followed by an identifier with no space in
                       // between.
     TYPE_INTEGER,     // A sequence of digits representing an integer.  Normally
                       // the digits are decimal, but a prefix of "0x" indicates
                       // a hex number and a leading zero indicates octal, just
                       // like with C numeric literals.  A leading negative sign
                       // is NOT included in the token; it's up to the parser to
                       // interpret the unary minus operator on its own.
     TYPE_FLOAT,       // A floating point literal, with a fractional part and/or
                       // an exponent.  Always in decimal.  Again, never
                       // negative.
     TYPE_STRING,      // A quoted sequence of escaped characters.  Either single
                       // or double quotes can be used, but they must match.
                       // A string literal cannot cross a line break.
     TYPE_SYMBOL,      // Any other printable character, like '!' or '+'.
                       // Symbols are always a single character, so "!+$%" is
                       // four tokens.
   };

   // Structure representing a token read from the token stream.
   struct Token {
     TokenType type;
     string text;       // The exact text of the token as it appeared in
                        // the input.  e.g. tokens of TYPE_STRING will still
                        // be escaped and in quotes.

     // "line" and "column" specify the position of the first character of
     // the token within the input stream.  They are zero-based.
     int line;
     int column;
     int end_column;
   };

   // Get the current token.  This is updated when Next() is called.  Before
   // the first call to Next(), current() has type TYPE_START and no contents.
   const Token& current();

   // Return the previous token -- i.e. what current() returned before the
   // previous call to Next().
   const Token& previous();

   // Advance to the next token.  Returns false if the end of the input is
   // reached.
   bool Next();

   // Like Next(), but also collects comments which appear between the previous
   // and next tokens.
   //
   // Comments which appear to be attached to the previous token are stored
   // in *prev_tailing_comments.  Comments which appear to be attached to the
   // next token are stored in *next_leading_comments.  Comments appearing in
   // between which do not appear to be attached to either will be added to
   // detached_comments.  Any of these parameters can be NULL to simply discard
   // the comments.
   //
   // A series of line comments appearing on consecutive lines, with no other
   // tokens appearing on those lines, will be treated as a single comment.
   //
   // Only the comment content is returned; comment markers (e.g. //) are
   // stripped out.  For block comments, leading whitespace and an asterisk will
   // be stripped from the beginning of each line other than the first.  Newlines
   // are included in the output.
   //
   // Examples:
   //
   //   optional int32 foo = 1;  // Comment attached to foo.
   //   // Comment attached to bar.
   //   optional int32 bar = 2;
   //
   //   optional string baz = 3;
   //   // Comment attached to baz.
   //   // Another line attached to baz.
   //
   //   // Comment attached to qux.
   //   //
   //   // Another line attached to qux.
   //   optional double qux = 4;
   //
   //   // Detached comment.  This is not attached to qux or corge
   //   // because there are blank lines separating it from both.
   //
   //   optional string corge = 5;
   //   /* Block comment attached
   //    * to corge.  Leading asterisks
   //    * will be removed. */
   //   /* Block comment attached to
   //    * grault. */
   //   optional int32 grault = 6;
   bool NextWithComments(string* prev_trailing_comments,
                         vector<string>* detached_comments,
                         string* next_leading_comments);

   // Parse helpers ---------------------------------------------------

   // Parses a TYPE_FLOAT token.  This never fails, so long as the text actually
   // comes from a TYPE_FLOAT token parsed by Tokenizer.  If it doesn't, the
   // result is undefined (possibly an assert failure).
   static double ParseFloat(const string& text);

   // Parses a TYPE_STRING token.  This never fails, so long as the text actually
   // comes from a TYPE_STRING token parsed by Tokenizer.  If it doesn't, the
   // result is undefined (possibly an assert failure).
   static void ParseString(const string& text, string* output);

   // Identical to ParseString, but appends to output.
   static void ParseStringAppend(const string& text, string* output);

   // Parses a TYPE_INTEGER token.  Returns false if the result would be
   // greater than max_value.  Otherwise, returns true and sets *output to the
   // result.  If the text is not from a Token of type TYPE_INTEGER originally
   // parsed by a Tokenizer, the result is undefined (possibly an assert
   // failure).
   static bool ParseInteger(const string& text, uint64 max_value,
                            uint64* output);

   // Options ---------------------------------------------------------

   // Set true to allow floats to be suffixed with the letter 'f'.  Tokens
   // which would otherwise be integers but which have the 'f' suffix will be
   // forced to be interpreted as floats.  For all other purposes, the 'f' is
   // ignored.
   void set_allow_f_after_float(bool value) { allow_f_after_float_ = value; }

   // Valid values for set_comment_style().
   enum CommentStyle {
     // Line comments begin with "//", block comments are delimited by "/*" and
     // "*/".
     CPP_COMMENT_STYLE,
     // Line comments begin with "#".  No way to write block comments.
     SH_COMMENT_STYLE
   };

   // Sets the comment style.
   void set_comment_style(CommentStyle style) { comment_style_ = style; }

   // Whether to require whitespace between a number and a field name.
   // Default is true. Do not use this; for Google-internal cleanup only.
   void set_require_space_after_number(bool require) {
     require_space_after_number_ = require;
   }

   // Whether to allow string literals to span multiple lines. Default is false.
   // Do not use this; for Google-internal cleanup only.
   void set_allow_multiline_strings(bool allow) {
     allow_multiline_strings_ = allow;
   }

   // External helper: validate an identifier.
   static bool IsIdentifier(const string& text);

   // -----------------------------------------------------------------
  private:
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Tokenizer);

   Token current_;           // Returned by current().
   Token previous_;          // Returned by previous().

   ZeroCopyInputStream* input_;
   ErrorCollector* error_collector_;

   char current_char_;       // == buffer_[buffer_pos_], updated by NextChar().
   const char* buffer_;      // Current buffer returned from input_.
   int buffer_size_;         // Size of buffer_.
   int buffer_pos_;          // Current position within the buffer.
   bool read_error_;         // Did we previously encounter a read error?

   // Line and column number of current_char_ within the whole input stream.
   int line_;
   int column_;

   // String to which text should be appended as we advance through it.
   // Call RecordTo(&str) to start recording and StopRecording() to stop.
   // E.g. StartToken() calls RecordTo(&current_.text).  record_start_ is the
   // position within the current buffer where recording started.
   string* record_target_;
   int record_start_;

   // Options.
   bool allow_f_after_float_;
   CommentStyle comment_style_;
   bool require_space_after_number_;
   bool allow_multiline_strings_;

   // Since we count columns we need to interpret tabs somehow.  We'll take
   // the standard 8-character definition for lack of any way to do better.
   static const int kTabWidth = 8;

   // -----------------------------------------------------------------
   // Helper methods.

   // Consume this character and advance to the next one.
   void NextChar();

   // Read a new buffer from the input.
   void Refresh();

   inline void RecordTo(string* target);
   inline void StopRecording();

   // Called when the current character is the first character of a new
   // token (not including whitespace or comments).
   inline void StartToken();
   // Called when the current character is the first character after the
   // end of the last token.  After this returns, current_.text will
   // contain all text consumed since StartToken() was called.
   inline void EndToken();

   // Convenience method to add an error at the current line and column.
   void AddError(const string& message) {
     error_collector_->AddError(line_, column_, message);
   }

   // -----------------------------------------------------------------
   // The following four methods are used to consume tokens of specific
   // types.  They are actually used to consume all characters *after*
   // the first, since the calling function consumes the first character
   // in order to decide what kind of token is being read.

   // Read and consume a string, ending when the given delimiter is
   // consumed.
   void ConsumeString(char delimiter);

   // Read and consume a number, returning TYPE_FLOAT or TYPE_INTEGER
   // depending on what was read.  This needs to know if the first
   // character was a zero in order to correctly recognize hex and octal
   // numbers.
   // It also needs to know if the first characted was a . to parse floating
   // point correctly.
   TokenType ConsumeNumber(bool started_with_zero, bool started_with_dot);

   // Consume the rest of a line.
   void ConsumeLineComment(string* content);
   // Consume until "*/".
   void ConsumeBlockComment(string* content);

   enum NextCommentStatus {
     // Started a line comment.
     LINE_COMMENT,

     // Started a block comment.
     BLOCK_COMMENT,

     // Consumed a slash, then realized it wasn't a comment.  current_ has
     // been filled in with a slash token.  The caller should return it.
     SLASH_NOT_COMMENT,

     // We do not appear to be starting a comment here.
     NO_COMMENT
   };

   // If we're at the start of a new comment, consume it and return what kind
   // of comment it is.
   NextCommentStatus TryConsumeCommentStart();

   // -----------------------------------------------------------------
   // These helper methods make the parsing code more readable.  The
   // "character classes" refered to are defined at the top of the .cc file.
   // Basically it is a C++ class with one method:
   //   static bool InClass(char c);
   // The method returns true if c is a member of this "class", like "Letter"
   // or "Digit".

   // Returns true if the current character is of the given character
   // class, but does not consume anything.
   template<typename CharacterClass>
   inline bool LookingAt();

   // If the current character is in the given class, consume it and return
   // true.  Otherwise return false.
   // e.g. TryConsumeOne<Letter>()
   template<typename CharacterClass>
   inline bool TryConsumeOne();

   // Like above, but try to consume the specific character indicated.
   inline bool TryConsume(char c);

   // Consume zero or more of the given character class.
   template<typename CharacterClass>
   inline void ConsumeZeroOrMore();

   // Consume one or more of the given character class or log the given
   // error message.
   // e.g. ConsumeOneOrMore<Digit>("Expected digits.");
   template<typename CharacterClass>
   inline void ConsumeOneOrMore(const char* error);
 };

 // inline methods ====================================================
 inline const Tokenizer::Token& Tokenizer::current() {
   return current_;
 }

 inline const Tokenizer::Token& Tokenizer::previous() {
   return previous_;
 }

 inline void Tokenizer::ParseString(const string& text, string* output) {
   output->clear();
   ParseStringAppend(text, output);
 }

 }  // namespace io
 }  // namespace protobuf

 }  // namespace google
 #endif  // GOOGLE_PROTOBUF_IO_TOKENIZER_H__
	// Protocol Buffers - Google's data interchange format
	// Copyright 2008 Google Inc. All rights reserved.
	// https://developers.google.com/protocol-buffers/
	//
	// Redistribution and use in source and binary forms, with or without
	// modification, are permitted provided that the following conditions are
	// met:
	//
	// * Redistributions of source code must retain the above copyright
	// notice, this list of conditions and the following disclaimer.
	// * Redistributions in binary form must reproduce the above
	// copyright notice, this list of conditions and the following disclaimer
	// in the documentation and/or other materials provided with the
	// distribution.
	// * Neither the name of Google Inc. nor the names of its
	// contributors may be used to endorse or promote products derived from
	// this software without specific prior written permission.
	//
	// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

	// Author: kenton@google.com (Kenton Varda)
	// Based on original Protocol Buffers design by
	// Sanjay Ghemawat, Jeff Dean, and others.
	//
	// Class for parsing tokenized text from a ZeroCopyInputStream.

	#ifndef GOOGLE_PROTOBUF_IO_TOKENIZER_H__
	#define GOOGLE_PROTOBUF_IO_TOKENIZER_H__

	#include <string>
	#include <vector>
	#include <google/protobuf/stubs/common.h>

	namespace google {
	namespace protobuf {
	namespace io {

	class ZeroCopyInputStream; // zero_copy_stream.h

	// Defined in this file.
	class ErrorCollector;
	class Tokenizer;

	// Abstract interface for an object which collects the errors that occur
	// during parsing. A typical implementation might simply print the errors
	// to stdout.
	class LIBPROTOBUF_EXPORT ErrorCollector {
	public:
	inline ErrorCollector() {}
	virtual ~ErrorCollector();

	// Indicates that there was an error in the input at the given line and
	// column numbers. The numbers are zero-based, so you may want to add
	// 1 to each before printing them.
	virtual void AddError(int line, int column, const string& message) = 0;

	// Indicates that there was a warning in the input at the given line and
	// column numbers. The numbers are zero-based, so you may want to add
	// 1 to each before printing them.
	virtual void AddWarning(int /* line /, int / column */,
	const string& /* message */) { }

	private:
	GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ErrorCollector);
	};

	// This class converts a stream of raw text into a stream of tokens for
	// the protocol definition parser to parse. The tokens recognized are
	// similar to those that make up the C language; see the TokenType enum for
	// precise descriptions. Whitespace and comments are skipped. By default,
	// C- and C++-style comments are recognized, but other styles can be used by
	// calling set_comment_style().
	class LIBPROTOBUF_EXPORT Tokenizer {
	public:
	// Construct a Tokenizer that reads and tokenizes text from the given
	// input stream and writes errors to the given error_collector.
	// The caller keeps ownership of input and error_collector.
	Tokenizer(ZeroCopyInputStream* input, ErrorCollector* error_collector);
	~Tokenizer();

	enum TokenType {
	TYPE_START, // Next() has not yet been called.
	TYPE_END, // End of input reached. "text" is empty.

	TYPE_IDENTIFIER, // A sequence of letters, digits, and underscores, not
	// starting with a digit. It is an error for a number
	// to be followed by an identifier with no space in
	// between.
	TYPE_INTEGER, // A sequence of digits representing an integer. Normally
	// the digits are decimal, but a prefix of "0x" indicates
	// a hex number and a leading zero indicates octal, just
	// like with C numeric literals. A leading negative sign
	// is NOT included in the token; it's up to the parser to
	// interpret the unary minus operator on its own.
	TYPE_FLOAT, // A floating point literal, with a fractional part and/or
	// an exponent. Always in decimal. Again, never
	// negative.
	TYPE_STRING, // A quoted sequence of escaped characters. Either single
	// or double quotes can be used, but they must match.
	// A string literal cannot cross a line break.
	TYPE_SYMBOL, // Any other printable character, like '!' or '+'.
	// Symbols are always a single character, so "!+$%" is
	// four tokens.
	};

	// Structure representing a token read from the token stream.
	struct Token {
	TokenType type;
	string text; // The exact text of the token as it appeared in
	// the input. e.g. tokens of TYPE_STRING will still
	// be escaped and in quotes.

	// "line" and "column" specify the position of the first character of
	// the token within the input stream. They are zero-based.
	int line;
	int column;
	int end_column;
	};

	// Get the current token. This is updated when Next() is called. Before
	// the first call to Next(), current() has type TYPE_START and no contents.
	const Token& current();

	// Return the previous token -- i.e. what current() returned before the
	// previous call to Next().
	const Token& previous();

	// Advance to the next token. Returns false if the end of the input is
	// reached.
	bool Next();

	// Like Next(), but also collects comments which appear between the previous
	// and next tokens.
	//
	// Comments which appear to be attached to the previous token are stored
	// in *prev_tailing_comments. Comments which appear to be attached to the
	// next token are stored in *next_leading_comments. Comments appearing in
	// between which do not appear to be attached to either will be added to
	// detached_comments. Any of these parameters can be NULL to simply discard
	// the comments.
	//
	// A series of line comments appearing on consecutive lines, with no other
	// tokens appearing on those lines, will be treated as a single comment.
	//
	// Only the comment content is returned; comment markers (e.g. //) are
	// stripped out. For block comments, leading whitespace and an asterisk will
	// be stripped from the beginning of each line other than the first. Newlines
	// are included in the output.
	//
	// Examples:
	//
	// optional int32 foo = 1; // Comment attached to foo.
	// // Comment attached to bar.
	// optional int32 bar = 2;
	//
	// optional string baz = 3;
	// // Comment attached to baz.
	// // Another line attached to baz.
	//
	// // Comment attached to qux.
	// //
	// // Another line attached to qux.
	// optional double qux = 4;
	//
	// // Detached comment. This is not attached to qux or corge
	// // because there are blank lines separating it from both.
	//
	// optional string corge = 5;
	// /* Block comment attached
	// * to corge. Leading asterisks
	// * will be removed. */
	// /* Block comment attached to
	// * grault. */
	// optional int32 grault = 6;
	bool NextWithComments(string* prev_trailing_comments,
	vector<string>* detached_comments,
	string* next_leading_comments);

	// Parse helpers ---------------------------------------------------

	// Parses a TYPE_FLOAT token. This never fails, so long as the text actually
	// comes from a TYPE_FLOAT token parsed by Tokenizer. If it doesn't, the
	// result is undefined (possibly an assert failure).
	static double ParseFloat(const string& text);

	// Parses a TYPE_STRING token. This never fails, so long as the text actually
	// comes from a TYPE_STRING token parsed by Tokenizer. If it doesn't, the
	// result is undefined (possibly an assert failure).
	static void ParseString(const string& text, string* output);

	// Identical to ParseString, but appends to output.
	static void ParseStringAppend(const string& text, string* output);

	// Parses a TYPE_INTEGER token. Returns false if the result would be
	// greater than max_value. Otherwise, returns true and sets *output to the
	// result. If the text is not from a Token of type TYPE_INTEGER originally
	// parsed by a Tokenizer, the result is undefined (possibly an assert
	// failure).
	static bool ParseInteger(const string& text, uint64 max_value,
	uint64* output);

	// Options ---------------------------------------------------------

	// Set true to allow floats to be suffixed with the letter 'f'. Tokens
	// which would otherwise be integers but which have the 'f' suffix will be
	// forced to be interpreted as floats. For all other purposes, the 'f' is
	// ignored.
	void set_allow_f_after_float(bool value) { allow_f_after_float_ = value; }

	// Valid values for set_comment_style().
	enum CommentStyle {
	// Line comments begin with "//", block comments are delimited by "/*" and
	// "*/".
	CPP_COMMENT_STYLE,
	// Line comments begin with "#". No way to write block comments.
	SH_COMMENT_STYLE
	};

	// Sets the comment style.
	void set_comment_style(CommentStyle style) { comment_style_ = style; }

	// Whether to require whitespace between a number and a field name.
	// Default is true. Do not use this; for Google-internal cleanup only.
	void set_require_space_after_number(bool require) {
	require_space_after_number_ = require;
	}

	// Whether to allow string literals to span multiple lines. Default is false.
	// Do not use this; for Google-internal cleanup only.
	void set_allow_multiline_strings(bool allow) {
	allow_multiline_strings_ = allow;
	}

	// External helper: validate an identifier.
	static bool IsIdentifier(const string& text);

	// -----------------------------------------------------------------
	private:
	GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Tokenizer);

	Token current_; // Returned by current().
	Token previous_; // Returned by previous().

	ZeroCopyInputStream* input_;
	ErrorCollector* error_collector_;

	char current_char_; // == buffer_[buffer_pos_], updated by NextChar().
	const char* buffer_; // Current buffer returned from input_.
	int buffer_size_; // Size of buffer_.
	int buffer_pos_; // Current position within the buffer.
	bool read_error_; // Did we previously encounter a read error?

	// Line and column number of current_char_ within the whole input stream.
	int line_;
	int column_;

	// String to which text should be appended as we advance through it.
	// Call RecordTo(&str) to start recording and StopRecording() to stop.
	// E.g. StartToken() calls RecordTo(&current_.text). record_start_ is the
	// position within the current buffer where recording started.
	string* record_target_;
	int record_start_;

	// Options.
	bool allow_f_after_float_;
	CommentStyle comment_style_;
	bool require_space_after_number_;
	bool allow_multiline_strings_;

	// Since we count columns we need to interpret tabs somehow. We'll take
	// the standard 8-character definition for lack of any way to do better.
	static const int kTabWidth = 8;

	// -----------------------------------------------------------------
	// Helper methods.

	// Consume this character and advance to the next one.
	void NextChar();

	// Read a new buffer from the input.
	void Refresh();

	inline void RecordTo(string* target);
	inline void StopRecording();

	// Called when the current character is the first character of a new
	// token (not including whitespace or comments).
	inline void StartToken();
	// Called when the current character is the first character after the
	// end of the last token. After this returns, current_.text will
	// contain all text consumed since StartToken() was called.
	inline void EndToken();

	// Convenience method to add an error at the current line and column.
	void AddError(const string& message) {
	error_collector_->AddError(line_, column_, message);
	}

	// -----------------------------------------------------------------
	// The following four methods are used to consume tokens of specific
	// types. They are actually used to consume all characters after
	// the first, since the calling function consumes the first character
	// in order to decide what kind of token is being read.

	// Read and consume a string, ending when the given delimiter is
	// consumed.
	void ConsumeString(char delimiter);

	// Read and consume a number, returning TYPE_FLOAT or TYPE_INTEGER
	// depending on what was read. This needs to know if the first
	// character was a zero in order to correctly recognize hex and octal
	// numbers.
	// It also needs to know if the first characted was a . to parse floating
	// point correctly.
	TokenType ConsumeNumber(bool started_with_zero, bool started_with_dot);

	// Consume the rest of a line.
	void ConsumeLineComment(string* content);
	// Consume until "*/".
	void ConsumeBlockComment(string* content);

	enum NextCommentStatus {
	// Started a line comment.
	LINE_COMMENT,

	// Started a block comment.
	BLOCK_COMMENT,

	// Consumed a slash, then realized it wasn't a comment. current_ has
	// been filled in with a slash token. The caller should return it.
	SLASH_NOT_COMMENT,

	// We do not appear to be starting a comment here.
	NO_COMMENT
	};

	// If we're at the start of a new comment, consume it and return what kind
	// of comment it is.
	NextCommentStatus TryConsumeCommentStart();

	// -----------------------------------------------------------------
	// These helper methods make the parsing code more readable. The
	// "character classes" refered to are defined at the top of the .cc file.
	// Basically it is a C++ class with one method:
	// static bool InClass(char c);
	// The method returns true if c is a member of this "class", like "Letter"
	// or "Digit".

	// Returns true if the current character is of the given character
	// class, but does not consume anything.
	template<typename CharacterClass>
	inline bool LookingAt();

	// If the current character is in the given class, consume it and return
	// true. Otherwise return false.
	// e.g. TryConsumeOne<Letter>()
	template<typename CharacterClass>
	inline bool TryConsumeOne();

	// Like above, but try to consume the specific character indicated.
	inline bool TryConsume(char c);

	// Consume zero or more of the given character class.
	template<typename CharacterClass>
	inline void ConsumeZeroOrMore();

	// Consume one or more of the given character class or log the given
	// error message.
	// e.g. ConsumeOneOrMore<Digit>("Expected digits.");
	template<typename CharacterClass>
	inline void ConsumeOneOrMore(const char* error);
	};

	// inline methods ====================================================
	inline const Tokenizer::Token& Tokenizer::current() {
	return current_;
	}

	inline const Tokenizer::Token& Tokenizer::previous() {
	return previous_;
	}

	inline void Tokenizer::ParseString(const string& text, string* output) {
	output->clear();
	ParseStringAppend(text, output);
	}

	} // namespace io
	} // namespace protobuf

	} // namespace google
	#endif // GOOGLE_PROTOBUF_IO_TOKENIZER_H__