Mercurial > pub > ImplabNet

comparison Implab/Formats/TextScanner.cs @ 174:983df35b3ca1 ref20160224

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
sync
author cin
date Fri, 18 Mar 2016 18:10:30 +0300
parents ecfece82ca11
children 96a89dcb4060
comparison
equal deleted inserted replaced
173:ecfece82ca11 174:983df35b3ca1
2 using Implab.Components; 2 using Implab.Components;
3 3
4 namespace Implab.Formats { 4 namespace Implab.Formats {
5 public abstract class TextScanner<TTag> : Disposable { 5 public abstract class TextScanner<TTag> : Disposable {
6 6
7 char[] m_buffer; 7 readonly int[] m_buffer;
8 int m_offset; 8 int m_bufferOffset;
9 int m_length; 9 int m_dataLength;
10 int m_tokenOffset;
11 int m_tokenLength; 10 int m_tokenLength;
11
12 TTag[] m_tags; 12 TTag[] m_tags;
13 13
14 BufferScanner<TTag> m_scanner; 14 BufferScanner<TTag> m_scanner;
15 15
16 protected bool ReadTokenInternal() { 16 protected bool ReadTokenInternal() {
17 if (EOF) 17 if (EOF)
18 return false; 18 return false;
19 19
20 // create a new scanner from template (scanners are structs) 20 // create a new scanner from template (scanners are value types)
21 var inst = m_scanner; 21 var inst = m_scanner;
22 22
23 // initialize the scanner 23 m_tokenLength = 0;
24 inst.Init(m_buffer, m_offset, m_length);
25 24
26 // do work 25 while (inst.Scan(m_buffer, m_bufferOffset, m_dataLength)) {
27 while (inst.Scan()) 26 m_tokenLength += m_dataLength;
28 Feed(ref inst); 27
28 var actual = Read(m_buffer, 0, m_buffer.Length);
29
30 m_bufferOffset = 0;
31 m_dataLength = actual;
32
33 if (actual == 0) {
34 inst.Eof();
35 break;
36 }
37 }
38
39 var len = inst.Position - m_bufferOffset;
40 m_tokenLength += len;
41 m_dataLength -= len;
42 m_bufferOffset = inst.Position;
29 43
30 // save result; 44 // save result;
31 m_buffer = inst.Buffer; 45
32 m_length = inst.Length;
33 m_offset = inst.Position;
34 m_tokenOffset = inst.TokenOffset;
35 m_tokenLength = inst.TokenLength;
36
37 m_tags = inst.GetTokenTags(); 46 m_tags = inst.GetTokenTags();
38 } 47 }
39 48
40 protected string GetToken() { 49 protected abstract int Read(int[] buffer, int offset, int size);
41 return new String(m_buffer, m_tokenOffset, m_tokenLength);
42 }
43 50
44 protected TTag[] Tags { 51 protected TTag[] Tags {
45 get { 52 get {
46 return m_tags; 53 return m_tags;
47 } 54 }
48 } 55 }
49 56
50 /// <summary>
51 /// Feed the specified scanner.
52 /// </summary>
53 /// <param name="scanner">Scanner.</param>
54 /// <example>
55 /// protected override void Feed(ref BufferScanner<TTag> scanner) {
56 /// var size = scanner.Extend();
57 /// var actual = m_reader.Read(scanner.Buffer, scanner.HiMark, size);
58 /// if (actual == 0) {
59 /// m_eof = true;
60 /// scanner.Eof();
61 /// } else {
62 /// scanner.RaiseHiMark(actual);
63 /// }
64 /// }
65 /// </example>
66 protected abstract void Feed(ref BufferScanner<TTag> scanner);
67
68 public abstract bool EOF { get; } 57 public abstract bool EOF { get; }
69 58
70 } 59 }
71 } 60 }
72 61