Mercurial > pub > ImplabNet
diff Implab/Formats/TextScanner.cs @ 174:983df35b3ca1 ref20160224
sync
author | cin |
---|---|
date | Fri, 18 Mar 2016 18:10:30 +0300 |
parents | ecfece82ca11 |
children | 96a89dcb4060 |
line wrap: on
line diff
--- a/Implab/Formats/TextScanner.cs Tue Mar 15 02:11:06 2016 +0300 +++ b/Implab/Formats/TextScanner.cs Fri Mar 18 18:10:30 2016 +0300 @@ -4,11 +4,11 @@ namespace Implab.Formats { public abstract class TextScanner<TTag> : Disposable { - char[] m_buffer; - int m_offset; - int m_length; - int m_tokenOffset; + readonly int[] m_buffer; + int m_bufferOffset; + int m_dataLength; int m_tokenLength; + TTag[] m_tags; BufferScanner<TTag> m_scanner; @@ -17,29 +17,36 @@ if (EOF) return false; - // create a new scanner from template (scanners are structs) + // create a new scanner from template (scanners are value types) var inst = m_scanner; - // initialize the scanner - inst.Init(m_buffer, m_offset, m_length); + m_tokenLength = 0; + + while (inst.Scan(m_buffer, m_bufferOffset, m_dataLength)) { + m_tokenLength += m_dataLength; + + var actual = Read(m_buffer, 0, m_buffer.Length); + + m_bufferOffset = 0; + m_dataLength = actual; - // do work - while (inst.Scan()) - Feed(ref inst); + if (actual == 0) { + inst.Eof(); + break; + } + } + + var len = inst.Position - m_bufferOffset; + m_tokenLength += len; + m_dataLength -= len; + m_bufferOffset = inst.Position; // save result; - m_buffer = inst.Buffer; - m_length = inst.Length; - m_offset = inst.Position; - m_tokenOffset = inst.TokenOffset; - m_tokenLength = inst.TokenLength; - + m_tags = inst.GetTokenTags(); } - protected string GetToken() { - return new String(m_buffer, m_tokenOffset, m_tokenLength); - } + protected abstract int Read(int[] buffer, int offset, int size); protected TTag[] Tags { get { @@ -47,26 +54,8 @@ } } - /// <summary> - /// Feed the specified scanner. - /// </summary> - /// <param name="scanner">Scanner.</param> - /// <example> - /// protected override void Feed(ref BufferScanner<TTag> scanner) { - /// var size = scanner.Extend(); - /// var actual = m_reader.Read(scanner.Buffer, scanner.HiMark, size); - /// if (actual == 0) { - /// m_eof = true; - /// scanner.Eof(); - /// } else { - /// scanner.RaiseHiMark(actual); - /// } - /// } - /// </example> - protected abstract void Feed(ref BufferScanner<TTag> scanner); - public abstract bool EOF { get; } - + } }