diff Implab/Formats/TextScanner.cs @ 174:983df35b3ca1 ref20160224

sync
author cin
date Fri, 18 Mar 2016 18:10:30 +0300
parents ecfece82ca11
children 96a89dcb4060
line wrap: on
line diff
--- a/Implab/Formats/TextScanner.cs	Tue Mar 15 02:11:06 2016 +0300
+++ b/Implab/Formats/TextScanner.cs	Fri Mar 18 18:10:30 2016 +0300
@@ -4,11 +4,11 @@
 namespace Implab.Formats {
     public abstract class TextScanner<TTag> : Disposable {
 
-        char[] m_buffer;
-        int m_offset;
-        int m_length;
-        int m_tokenOffset;
+        readonly int[] m_buffer;
+        int m_bufferOffset;
+        int m_dataLength;
         int m_tokenLength;
+
         TTag[] m_tags;
 
         BufferScanner<TTag> m_scanner;
@@ -17,29 +17,36 @@
             if (EOF)
                 return false;
             
-            // create a new scanner from template (scanners are structs)
+            // create a new scanner from template (scanners are value types)
             var inst = m_scanner;
 
-            // initialize the scanner
-            inst.Init(m_buffer, m_offset, m_length);
+            m_tokenLength = 0;
+
+            while (inst.Scan(m_buffer, m_bufferOffset, m_dataLength)) {
+                m_tokenLength += m_dataLength;
+
+                var actual = Read(m_buffer, 0, m_buffer.Length);
+                    
+                m_bufferOffset = 0;
+                m_dataLength = actual;
 
-            // do work
-            while (inst.Scan())
-                Feed(ref inst);
+                if (actual == 0) {
+                    inst.Eof();
+                    break;
+                }
+            }
+
+            var len = inst.Position - m_bufferOffset;
+            m_tokenLength += len;
+            m_dataLength -= len;
+            m_bufferOffset = inst.Position;
 
             // save result;
-            m_buffer = inst.Buffer;
-            m_length = inst.Length;
-            m_offset = inst.Position;
-            m_tokenOffset = inst.TokenOffset;
-            m_tokenLength = inst.TokenLength;
-
+        
             m_tags = inst.GetTokenTags();
         }
 
-        protected string GetToken() {
-            return new String(m_buffer, m_tokenOffset, m_tokenLength);
-        }
+        protected abstract int Read(int[] buffer, int offset, int size);
 
         protected TTag[] Tags {
             get {
@@ -47,26 +54,8 @@
             }
         }
 
-        /// <summary>
-        /// Feed the specified scanner.
-        /// </summary>
-        /// <param name="scanner">Scanner.</param>
-        /// <example>
-        /// protected override void Feed(ref BufferScanner<TTag> scanner) {
-        ///     var size = scanner.Extend();
-        ///     var actual = m_reader.Read(scanner.Buffer, scanner.HiMark, size);
-        ///     if (actual == 0) {
-        ///         m_eof = true;
-        ///         scanner.Eof();
-        ///     } else {
-        ///         scanner.RaiseHiMark(actual);
-        ///     }
-        /// }
-        /// </example>
-        protected abstract void Feed(ref BufferScanner<TTag> scanner);
-
         public abstract bool EOF { get; }
-
+ 
     }
 }