Mercurial > pub > ImplabNet

comparison Implab/Formats/TextScanner.cs @ 182:76e8f2ba12b8 ref20160224

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
pretty print DFA, the minimization is still buggy
author cin
date Thu, 24 Mar 2016 18:52:10 +0300
parents b2b6a6640aa3
children
comparison
equal deleted inserted replaced
181:b2b6a6640aa3 182:76e8f2ba12b8
51 internal bool ReadToken<TTag>(int[,] dfa, bool[] final, TTag[][] tags, int state, int[] alphabet, out TTag[] tag) { 51 internal bool ReadToken<TTag>(int[,] dfa, bool[] final, TTag[][] tags, int state, int[] alphabet, out TTag[] tag) {
52 m_tokenLength = 0; 52 m_tokenLength = 0;
53 tag = null; 53 tag = null;
54 54
55 var maxSymbol = alphabet.Length - 1; 55 var maxSymbol = alphabet.Length - 1;
56 56 int next;
57 do { 57 do {
58 // after the next chunk is read the offset in the buffer may change 58 // after the next chunk is read the offset in the buffer may change
59 int pos = m_bufferOffset + m_tokenLength; 59 int pos = m_bufferOffset + m_tokenLength;
60 60 next = state;
61 while (pos < m_bufferSize) { 61 while (pos < m_bufferSize) {
62 var ch = m_buffer[pos]; 62 var ch = m_buffer[pos];
63 63
64 try { 64 next = dfa[next, ch > maxSymbol ? AutomatonConst.UNCLASSIFIED_INPUT : alphabet[ch]];
65 var next = dfa[state, ch > maxSymbol ? AutomatonConst.UNCLASSIFIED_INPUT : alphabet[ch]];
66 65
67 if (next == AutomatonConst.UNREACHABLE_STATE) 66 if (next == AutomatonConst.UNREACHABLE_STATE)
68 break; 67 break;
69 68
70 state = next; 69 state = next;
71 }catch {
72 throw;
73 }
74 pos++; 70 pos++;
75 } 71 }
76
77 m_tokenLength = pos - m_bufferOffset; 72 m_tokenLength = pos - m_bufferOffset;
78 } while (state != AutomatonConst.UNREACHABLE_STATE && Feed()); 73 } while (next != AutomatonConst.UNREACHABLE_STATE && Feed());
79 74
80 m_tokenOffset = m_bufferOffset; 75 m_tokenOffset = m_bufferOffset;
81 m_bufferOffset += m_tokenLength; 76 m_bufferOffset += m_tokenLength;
82 77
83 if (final[state]) { 78 if (final[state]) {
148 public string GetTokenValue() { 143 public string GetTokenValue() {
149 return new String(m_buffer, m_tokenOffset, m_tokenLength); 144 return new String(m_buffer, m_tokenOffset, m_tokenLength);
150 } 145 }
151 146
152 public void CopyTokenTo(char[] buffer, int offset) { 147 public void CopyTokenTo(char[] buffer, int offset) {
153 m_buffer.CopyTo(buffer, offset); 148 Array.Copy(m_buffer, m_tokenOffset,buffer, offset, m_tokenLength);
154 } 149 }
155 150
156 public void CopyTokenTo(StringBuilder sb) { 151 public void CopyTokenTo(StringBuilder sb) {
157 sb.Append(m_buffer, m_tokenOffset, m_tokenLength); 152 sb.Append(m_buffer, m_tokenOffset, m_tokenLength);
158 } 153 }