diff Implab/Formats/TextScanner.cs @ 180:c32688129f14 ref20160224

refactoring complete, JSONParser rewritten
author cin
date Thu, 24 Mar 2016 02:30:46 +0300
parents d5c5db0335ee
children b2b6a6640aa3
line wrap: on
line diff
--- a/Implab/Formats/TextScanner.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Formats/TextScanner.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -49,8 +49,8 @@
         /// <param name="alphabet"></param>
         /// <param name = "tag"></param>
         internal bool ReadToken<TTag>(int[,] dfa, bool[] final, TTag[][] tags, int state, int[] alphabet, out TTag[] tag) {
-            Safe.ArgumentNotNull();
             m_tokenLength = 0;
+            tag = null;
 
             var maxSymbol = alphabet.Length - 1;
 
@@ -109,7 +109,7 @@
                     var size = used + free;
 
                     if (size > m_bufferMax)
-                        throw new ParserException(String.Format("The buffer limit ({0} Kb) is reached", m_bufferMax/1024));
+                        throw new ParserException(String.Format("The buffer limit ({0} Kb) is reached", m_bufferMax / 1024));
                     
                     var temp = new char[size];
 
@@ -122,7 +122,13 @@
                     m_bufferOffset = 0;
                     m_bufferSize = used + read;
                     m_buffer = temp;
+                } else {
+                    var read = Read(m_buffer, m_bufferSize, m_chunkSize);
+                    if (read == 0)
+                        return false;
+                    m_bufferSize += m_chunkSize;
                 }
+                return true;
             } else {
                 Debug.Assert(m_bufferOffset == 0);
                 m_buffer = new char[m_chunkSize];