1 files changed, 53 insertions, 0 deletions
diff --git a/token.h b/token.h
new file mode 100644
index 0000000..3decdbc
--- /dev/null
+++ b/token.h
@@ -0,0 +1,53 @@
+#ifndef INCLUDE_BLASSIC_TOKEN_H
+#define INCLUDE_BLASSIC_TOKEN_H
+
+// token.h
+// Revision 7-feb-2005
+
+
+#include "blassic.h"
+
+#include <string>
+
+
+class Tokenizer {
+public:
+	enum Type { Empty, EndLine, Blank, Plain, Literal, Integer };
+	Tokenizer (const std::string & source);
+
+	// Members of token are public accessible for simplicity.
+	// Use with care.
+	class Token {
+	public:
+		Token () :
+			type (Empty)
+		{ }
+		Token (Type t) :
+			type (t)
+		{ }
+		Token (Type t, const std::string & str) :
+			type (t), str (str)
+		{ }
+		Token (BlInteger n) :
+			type (Integer), n (n)
+		{ }
+		Type type;
+		std::string str;
+		BlInteger n;
+	};
+	Token get ();
+	std::string getrest ();
+	unsigned char peek ();
+private:
+	unsigned char peeknospace ();
+	unsigned char nextchar ();
+	unsigned char nextcharnospace ();
+	void ungetchar ();
+	std::string str;
+	std::string::size_type pos;
+	const std::string::size_type limit;
+};
+
+#endif
+
+// Fin de token.h