Mercurial > pub > ImplabNet

comparison Implab/Formats/TextScanner.cs @ 180:c32688129f14 ref20160224

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
refactoring complete, JSONParser rewritten
author cin
date Thu, 24 Mar 2016 02:30:46 +0300
parents d5c5db0335ee
children b2b6a6640aa3
comparison
equal deleted inserted replaced
179:478ef706906a 180:c32688129f14
47 /// <param name="tags">Tags.</param> 47 /// <param name="tags">Tags.</param>
48 /// <param name="state">The initial state for the automaton.</param> 48 /// <param name="state">The initial state for the automaton.</param>
49 /// <param name="alphabet"></param> 49 /// <param name="alphabet"></param>
50 /// <param name = "tag"></param> 50 /// <param name = "tag"></param>
51 internal bool ReadToken<TTag>(int[,] dfa, bool[] final, TTag[][] tags, int state, int[] alphabet, out TTag[] tag) { 51 internal bool ReadToken<TTag>(int[,] dfa, bool[] final, TTag[][] tags, int state, int[] alphabet, out TTag[] tag) {
52 Safe.ArgumentNotNull();
53 m_tokenLength = 0; 52 m_tokenLength = 0;
53 tag = null;
54 54
55 var maxSymbol = alphabet.Length - 1; 55 var maxSymbol = alphabet.Length - 1;
56 56
57 do { 57 do {
58 // after the next chunk is read the offset in the buffer may change 58 // after the next chunk is read the offset in the buffer may change
107 free += m_chunkSize; 107 free += m_chunkSize;
108 var used = m_bufferSize - m_bufferOffset; 108 var used = m_bufferSize - m_bufferOffset;
109 var size = used + free; 109 var size = used + free;
110 110
111 if (size > m_bufferMax) 111 if (size > m_bufferMax)
112 throw new ParserException(String.Format("The buffer limit ({0} Kb) is reached", m_bufferMax/1024)); 112 throw new ParserException(String.Format("The buffer limit ({0} Kb) is reached", m_bufferMax / 1024));
113 113
114 var temp = new char[size]; 114 var temp = new char[size];
115 115
116 var read = Read(temp, used, m_chunkSize); 116 var read = Read(temp, used, m_chunkSize);
117 if (read == 0) 117 if (read == 0)
120 Array.Copy(m_buffer, m_bufferOffset, temp, 0, used); 120 Array.Copy(m_buffer, m_bufferOffset, temp, 0, used);
121 121
122 m_bufferOffset = 0; 122 m_bufferOffset = 0;
123 m_bufferSize = used + read; 123 m_bufferSize = used + read;
124 m_buffer = temp; 124 m_buffer = temp;
125 } else {
126 var read = Read(m_buffer, m_bufferSize, m_chunkSize);
127 if (read == 0)
128 return false;
129 m_bufferSize += m_chunkSize;
125 } 130 }
131 return true;
126 } else { 132 } else {
127 Debug.Assert(m_bufferOffset == 0); 133 Debug.Assert(m_bufferOffset == 0);
128 m_buffer = new char[m_chunkSize]; 134 m_buffer = new char[m_chunkSize];
129 m_bufferSize = Read(m_buffer, 0, m_chunkSize); 135 m_bufferSize = Read(m_buffer, 0, m_chunkSize);
130 return (m_bufferSize != 0); 136 return (m_bufferSize != 0);