Mercurial > pub > ImplabNet

comparison Implab/Formats/JSON/JsonScanner.cs @ 229:5f7a3e1d32b9 v2

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
JsonXmlReader performance tuning JsonScanner now operates strings and doesn't parses number and literals. Added SerializationHelpers to common serialize/deserialize operations
author cin
date Tue, 12 Sep 2017 19:07:42 +0300
parents 6fa235c5a760
children
comparison
equal deleted inserted replaced
228:6fa235c5a760 229:5f7a3e1d32b9
23 m_buffer = buffer; 23 m_buffer = buffer;
24 m_pos = pos; 24 m_pos = pos;
25 m_length = length; 25 m_length = length;
26 } 26 }
27 27
28 bool Read(InputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) { 28 bool ReadChunk(InputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) {
29 scanner.Reset(); 29 scanner.ResetState();
30 30
31 if (m_pos == m_length) { 31 while(scanner.Scan(m_buffer, m_pos, m_length)) {
32 // scanner requests new data
33
34 if (m_pos != m_length) // capture results for the future
35 m_tokenBuilder.Append(m_buffer, m_pos, m_length - m_pos);
36
37 // read next data
38 m_length = Read(m_buffer, 0, m_buffer.Length);
39
40 if (m_length == 0) {
41 // no data is read
42 if (scanner.Position == m_pos) {
43 // scanned hasn't moved, that's the end
44 m_pos = 0;
45 tokenType = JsonGrammar.TokenType.None;
46 return false;
47 }
48
49 if (scanner.IsFinal) {
50 m_pos = 0;
51 tokenType = scanner.Tag;
52 return true;
53 } else {
54 throw new ParserException("Unexpected EOF");
55 }
56 }
57
32 m_pos = 0; 58 m_pos = 0;
33 m_length = Read(m_buffer, 0, m_buffer.Length);
34 if (m_length == 0) {
35 tokenType = JsonGrammar.TokenType.None;
36 return false; // EOF
37 }
38 }
39
40 while(scanner.Scan(m_buffer, m_pos, m_length - m_pos)) {
41 m_tokenBuilder.Append(m_buffer, m_pos, m_length - m_pos);
42 m_pos = 0;
43 m_length = Read(m_buffer, 0, m_buffer.Length);
44 } 59 }
45 var scannerPos = scanner.Position; 60 var scannerPos = scanner.Position;
61
62 // scanner stops as scannerPos
63 if (!scanner.IsFinal)
64 throw new ParserException($"Unexpected character '{m_buffer[scannerPos + 1]}'");
65
66 tokenType = scanner.Tag;
67 if (scannerPos != m_pos && tokenType == JsonGrammar.TokenType.Number || tokenType == JsonGrammar.TokenType.Literal)
68 m_tokenBuilder.Append(m_buffer, m_pos, scannerPos - m_pos);
69
70 m_pos = scannerPos;
71 return true;
72 }
73
74 bool ReadStringChunk(InputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) {
75 scanner.ResetState();
76
77 while (scanner.Scan(m_buffer, m_pos, m_length)) {
78 // scanner requests new data
79
80 if (m_pos != m_length) // capture results for the future
81 m_tokenBuilder.Append(m_buffer, m_pos, m_length - m_pos);
82
83 // read next data
84 m_length = Read(m_buffer, 0, m_buffer.Length);
85
86 if (m_length == 0) {
87 // no data is read
88 if (scanner.Position == m_pos) {
89 // scanned hasn't moved, that's the end
90 m_pos = 0;
91 tokenType = JsonGrammar.TokenType.None;
92 return false;
93 }
94
95 if (scanner.IsFinal) {
96 m_pos = 0;
97 tokenType = scanner.Tag;
98 return true;
99 } else {
100 throw new ParserException("Unexpected EOF");
101 }
102 }
103
104 m_pos = 0;
105 }
106 var scannerPos = scanner.Position;
107
108 // scanner stops as scannerPos
109 if (!scanner.IsFinal)
110 throw new ParserException($"Unexpected character '{m_buffer[scannerPos + 1]}'");
111
46 if (scannerPos != m_pos) { 112 if (scannerPos != m_pos) {
47 m_tokenBuilder.Append(m_buffer, m_pos, scannerPos - m_pos); 113 m_tokenBuilder.Append(m_buffer, m_pos, scannerPos - m_pos);
48 m_pos = scannerPos; 114 m_pos = scannerPos;
49 }
50
51 if (!scanner.IsFinal) {
52 if (m_length == 0) {
53 // unexpected EOF
54 throw new ParserException("Unexpected EOF");
55 } else {
56 // unecpected character
57 throw new ParserException($"Unexpected character '{m_buffer[m_pos + 1]}'");
58 }
59 } 115 }
60 tokenType = scanner.Tag; 116 tokenType = scanner.Tag;
61 return true; 117 return true;
62 } 118 }
63 119
70 /// <param name="tokenValue">Возвращает значение прочитанного токена.</param> 126 /// <param name="tokenValue">Возвращает значение прочитанного токена.</param>
71 /// <param name="tokenType">Возвращает тип прочитанного токена.</param> 127 /// <param name="tokenType">Возвращает тип прочитанного токена.</param>
72 /// <returns><c>true</c> - чтение произведено успешно. <c>false</c> - достигнут конец входных данных</returns> 128 /// <returns><c>true</c> - чтение произведено успешно. <c>false</c> - достигнут конец входных данных</returns>
73 /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е. 129 /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е.
74 /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks> 130 /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks>
75 public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) { 131 public bool ReadToken(out string tokenValue, out JsonTokenType tokenType) {
76 JsonGrammar.TokenType tag; 132 JsonGrammar.TokenType tag;
77 m_tokenBuilder.Clear(); 133 m_tokenBuilder.Clear();
78 while (Read(m_jsonContext, out tag)) { 134 while (ReadChunk(m_jsonContext, out tag)) {
79 switch (tag) { 135 switch (tag) {
80 case JsonGrammar.TokenType.StringBound: 136 case JsonGrammar.TokenType.StringBound:
81 tokenValue = ReadString(); 137 tokenValue = ReadString();
82 tokenType = JsonTokenType.String; 138 tokenType = JsonTokenType.String;
83 break; 139 break;
84 case JsonGrammar.TokenType.Number: 140 case JsonGrammar.TokenType.Number:
85 tokenValue = Double.Parse(m_tokenBuilder.ToString(), CultureInfo.InvariantCulture); 141 tokenValue = m_tokenBuilder.ToString();
86 tokenType = JsonTokenType.Number; 142 tokenType = JsonTokenType.Number;
87 break; 143 break;
88 case JsonGrammar.TokenType.Literal: 144 case JsonGrammar.TokenType.Literal:
89 tokenType = JsonTokenType.Literal; 145 tokenType = JsonTokenType.Literal;
90 tokenValue = m_tokenBuilder.ToString(); 146 tokenValue = m_tokenBuilder.ToString();
106 162
107 string ReadString() { 163 string ReadString() {
108 JsonGrammar.TokenType tag; 164 JsonGrammar.TokenType tag;
109 m_tokenBuilder.Clear(); 165 m_tokenBuilder.Clear();
110 166
111 while (Read(m_stringContext, out tag)) { 167 while (ReadStringChunk(m_stringContext, out tag)) {
112 switch (tag) { 168 switch (tag) {
113 case JsonGrammar.TokenType.StringBound: 169 case JsonGrammar.TokenType.StringBound:
114 m_tokenBuilder.Length--; 170 m_tokenBuilder.Length--;
115 return m_tokenBuilder.ToString(); 171 return m_tokenBuilder.ToString();
116 case JsonGrammar.TokenType.UnescapedChar: 172 case JsonGrammar.TokenType.UnescapedChar: