larkdown/Parser/Lexer/Lexer.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;

namespace Larkdown.Parser.Lexer
{
    public class Lexer
    {
		private readonly string _source;
		private readonly Queue<Token> _buffer;
		private int _position = 0;

		public Lexer(string source)
		{
			_source = Regex.Replace(source, @"\r\n|\n\r|\n|\r", "\n");
			_buffer = new Queue<Token>();
		}

		public IEnumerable<Token> Tokens()
		{
			State state = new LineStartState();

			while (state != null)
			{
				state = state.Next(this);

				while (_buffer.Count > 0)
				{
					yield return _buffer.Dequeue();
				}
			}

			yield break;
		}

		public Lexer Emit(Token token)
		{
			_buffer.Enqueue(token);
			return this;
		}

		public char Read()
		{
			return _source[_position++];
		}

		public string Read(Regex exp)
		{
			var match = exp.Match(_source, _position);
			_position += match.Length;

			return match.Success ? match.Captures[0].Value : String.Empty;
		}

		public char Peek()
		{
			return _source[_position];
		}

		public bool IsEof()
		{
			return _position == _source.Length;
		}
	}
}