Mercurial > pub > ImplabNet

annotate Implab/Formats/TextScanner.cs @ 175:96a89dcb4060 ref20160224

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
sync
author cin
date Mon, 21 Mar 2016 18:41:45 +0300
parents 983df35b3ca1
children 0c3c69fe225b
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
173
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
1 using System;
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
2 using Implab.Components;
175
cin
parents: 174
diff changeset
3 using Implab.Automaton.RegularExpressions;
cin
parents: 174
diff changeset
4 using System.Diagnostics;
cin
parents: 174
diff changeset
5 using Implab.Automaton;
173
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
6
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
7 namespace Implab.Formats {
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
8 public abstract class TextScanner<TTag> : Disposable {
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
9
175
cin
parents: 174
diff changeset
10 int m_maxSymbol;
cin
parents: 174
diff changeset
11 int[] m_symbolMap;
cin
parents: 174
diff changeset
12
cin
parents: 174
diff changeset
13 readonly char[] m_buffer;
174
cin
parents: 173
diff changeset
14 int m_bufferOffset;
175
cin
parents: 174
diff changeset
15 int m_bufferSize;
173
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
16 int m_tokenLength;
174
cin
parents: 173
diff changeset
17
173
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
18 TTag[] m_tags;
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
19
175
cin
parents: 174
diff changeset
20 protected bool ReadTokenInternal(DFAStateDescriptor<TTag>[] dfa, int state) {
cin
parents: 174
diff changeset
21 Debug.Assert(dfa != null);
174
cin
parents: 173
diff changeset
22
175
cin
parents: 174
diff changeset
23 do {
cin
parents: 174
diff changeset
24 for (var pos = m_bufferOffset; pos < m_bufferSize; pos++) {
cin
parents: 174
diff changeset
25 var ch = m_buffer[pos];
cin
parents: 174
diff changeset
26 state = dfa[state].transitions[m_symbolMap[ch > m_maxSymbol ? m_maxSymbol : ch]];
cin
parents: 174
diff changeset
27 if (state == DFAConst.UNREACHABLE_STATE)
cin
parents: 174
diff changeset
28 break;
cin
parents: 174
diff changeset
29 }
cin
parents: 174
diff changeset
30 } while (Feed());
174
cin
parents: 173
diff changeset
31
175
cin
parents: 174
diff changeset
32 if (dfa[state].final) {
173
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
33
174
cin
parents: 173
diff changeset
34 }
cin
parents: 173
diff changeset
35
173
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
36 }
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
37
175
cin
parents: 174
diff changeset
38 bool Feed() {
cin
parents: 174
diff changeset
39
cin
parents: 174
diff changeset
40 }
cin
parents: 174
diff changeset
41
cin
parents: 174
diff changeset
42 protected abstract int Read(char[] buffer, int offset, int size);
173
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
43
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
44 protected TTag[] Tags {
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
45 get {
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
46 return m_tags;
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
47 }
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
48 }
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
49
175
cin
parents: 174
diff changeset
50
173
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
51 }
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
52 }
ecfece82ca11 Working on text scanner
cin
parents:
diff changeset
53