|
165
|
1 using System;
|
|
163
|
2 using System.Globalization;
|
|
165
|
3 using Implab.Automaton;
|
|
176
|
4 using System.Text;
|
|
|
5 using Implab.Components;
|
|
|
6 using System.IO;
|
|
163
|
7
|
|
228
|
8 namespace Implab.Formats.Json {
|
|
163
|
9 /// <summary>
|
|
|
10 /// Сканнер (лексер), разбивающий поток символов на токены JSON.
|
|
|
11 /// </summary>
|
|
228
|
12 public abstract class JsonScanner : Disposable {
|
|
|
13 readonly InputScanner<JsonGrammar.TokenType> m_jsonContext = JsonGrammar.CreateJsonExpressionScanner();
|
|
|
14 readonly InputScanner<JsonGrammar.TokenType> m_stringContext = JsonGrammar.CreateStringExpressionScanner();
|
|
176
|
15
|
|
228
|
16 readonly char[] m_unescapeBuf = new char[4];
|
|
|
17 readonly char[] m_buffer;
|
|
|
18 int m_length;
|
|
|
19 int m_pos;
|
|
|
20 readonly StringBuilder m_tokenBuilder = new StringBuilder();
|
|
163
|
21
|
|
228
|
22 protected JsonScanner(char[] buffer, int pos, int length) {
|
|
|
23 m_buffer = buffer;
|
|
|
24 m_pos = pos;
|
|
|
25 m_length = length;
|
|
176
|
26 }
|
|
|
27
|
|
228
|
28 bool Read(InputScanner<JsonGrammar.TokenType> scanner, out JsonGrammar.TokenType tokenType) {
|
|
|
29 scanner.Reset();
|
|
176
|
30
|
|
228
|
31 if (m_pos == m_length) {
|
|
|
32 m_pos = 0;
|
|
|
33 m_length = Read(m_buffer, 0, m_buffer.Length);
|
|
|
34 if (m_length == 0) {
|
|
|
35 tokenType = JsonGrammar.TokenType.None;
|
|
|
36 return false; // EOF
|
|
|
37 }
|
|
|
38 }
|
|
|
39
|
|
|
40 while(scanner.Scan(m_buffer, m_pos, m_length - m_pos)) {
|
|
|
41 m_tokenBuilder.Append(m_buffer, m_pos, m_length - m_pos);
|
|
|
42 m_pos = 0;
|
|
|
43 m_length = Read(m_buffer, 0, m_buffer.Length);
|
|
|
44 }
|
|
|
45 var scannerPos = scanner.Position;
|
|
|
46 if (scannerPos != m_pos) {
|
|
|
47 m_tokenBuilder.Append(m_buffer, m_pos, scannerPos - m_pos);
|
|
|
48 m_pos = scannerPos;
|
|
|
49 }
|
|
|
50
|
|
|
51 if (!scanner.IsFinal) {
|
|
|
52 if (m_length == 0) {
|
|
|
53 // unexpected EOF
|
|
|
54 throw new ParserException("Unexpected EOF");
|
|
|
55 } else {
|
|
|
56 // unecpected character
|
|
|
57 throw new ParserException($"Unexpected character '{m_buffer[m_pos + 1]}'");
|
|
|
58 }
|
|
|
59 }
|
|
|
60 tokenType = scanner.Tag;
|
|
|
61 return true;
|
|
163
|
62 }
|
|
|
63
|
|
228
|
64 protected abstract int Read(char[] buffer, int offset, int size);
|
|
|
65
|
|
180
|
66
|
|
163
|
67 /// <summary>
|
|
|
68 /// Читает следующий лексический элемент из входных данных.
|
|
|
69 /// </summary>
|
|
|
70 /// <param name="tokenValue">Возвращает значение прочитанного токена.</param>
|
|
|
71 /// <param name="tokenType">Возвращает тип прочитанного токена.</param>
|
|
|
72 /// <returns><c>true</c> - чтение произведено успешно. <c>false</c> - достигнут конец входных данных</returns>
|
|
|
73 /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е.
|
|
|
74 /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks>
|
|
|
75 public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) {
|
|
228
|
76 JsonGrammar.TokenType tag;
|
|
|
77 m_tokenBuilder.Clear();
|
|
|
78 while (Read(m_jsonContext, out tag)) {
|
|
|
79 switch (tag) {
|
|
|
80 case JsonGrammar.TokenType.StringBound:
|
|
163
|
81 tokenValue = ReadString();
|
|
|
82 tokenType = JsonTokenType.String;
|
|
|
83 break;
|
|
228
|
84 case JsonGrammar.TokenType.Number:
|
|
|
85 tokenValue = Double.Parse(m_tokenBuilder.ToString(), CultureInfo.InvariantCulture);
|
|
163
|
86 tokenType = JsonTokenType.Number;
|
|
|
87 break;
|
|
228
|
88 case JsonGrammar.TokenType.Literal:
|
|
|
89 tokenType = JsonTokenType.Literal;
|
|
|
90 tokenValue = m_tokenBuilder.ToString();
|
|
|
91 break;
|
|
|
92 case JsonGrammar.TokenType.Whitespace:
|
|
|
93 m_tokenBuilder.Clear();
|
|
183
|
94 continue;
|
|
163
|
95 default:
|
|
228
|
96 tokenType = (JsonTokenType)tag;
|
|
|
97 tokenValue = null;
|
|
163
|
98 break;
|
|
|
99 }
|
|
|
100 return true;
|
|
|
101 }
|
|
|
102 tokenValue = null;
|
|
|
103 tokenType = JsonTokenType.None;
|
|
|
104 return false;
|
|
|
105 }
|
|
|
106
|
|
|
107 string ReadString() {
|
|
228
|
108 JsonGrammar.TokenType tag;
|
|
|
109 m_tokenBuilder.Clear();
|
|
176
|
110
|
|
228
|
111 while (Read(m_stringContext, out tag)) {
|
|
|
112 switch (tag) {
|
|
|
113 case JsonGrammar.TokenType.StringBound:
|
|
|
114 m_tokenBuilder.Length--;
|
|
|
115 return m_tokenBuilder.ToString();
|
|
|
116 case JsonGrammar.TokenType.UnescapedChar:
|
|
163
|
117 break;
|
|
228
|
118 case JsonGrammar.TokenType.EscapedUnicode: // \xXXXX - unicode escape sequence
|
|
|
119 m_tokenBuilder.CopyTo(m_tokenBuilder.Length - 4, m_unescapeBuf, 0, 4);
|
|
|
120 m_tokenBuilder.Length -= 6;
|
|
|
121 m_tokenBuilder.Append(StringTranslator.TranslateHexUnicode(m_unescapeBuf, 0));
|
|
163
|
122 break;
|
|
228
|
123 case JsonGrammar.TokenType.EscapedChar: // \t - escape sequence
|
|
|
124 var ch = m_tokenBuilder[m_tokenBuilder.Length-1];
|
|
|
125 m_tokenBuilder.Length -= 2;
|
|
|
126 m_tokenBuilder.Append(StringTranslator.TranslateEscapedChar(ch));
|
|
163
|
127 break;
|
|
|
128 }
|
|
|
129 }
|
|
|
130
|
|
|
131 throw new ParserException("Unexpected end of data");
|
|
|
132 }
|
|
|
133 }
|
|
|
134 }
|
