annotate Implab/JSON/JSONScanner.cs @ 119:2573b562e328 v2

Promises rewritten, added improved version of AsyncQueue
author cin
date Sun, 11 Jan 2015 19:13:02 +0300
parents 10c7337d29e7
children 130781364799
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
55
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
1 using Implab.Parsing;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
2 using System;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
3 using System.Collections.Generic;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
4 using System.Globalization;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
5 using System.Linq;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
6 using System.Text;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
7 using System.Threading.Tasks;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
8
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
9 namespace Implab.JSON {
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
10 /// <summary>
60
10c7337d29e7 +JSONXmlReaderOptions
cin
parents: 55
diff changeset
11 /// Сканнер (лексер), разбивающий поток символов на токены JSON.
55
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
12 /// </summary>
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
13 public class JSONScanner : Scanner {
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
14 char[] m_stringBuffer;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
15 DFAStateDescriptior[] m_stringDFA;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
16 int[] m_stringAlphabet;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
17
60
10c7337d29e7 +JSONXmlReaderOptions
cin
parents: 55
diff changeset
18 /// <summary>
10c7337d29e7 +JSONXmlReaderOptions
cin
parents: 55
diff changeset
19 /// Создает новый экземпляр сканнера
10c7337d29e7 +JSONXmlReaderOptions
cin
parents: 55
diff changeset
20 /// </summary>
55
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
21 public JSONScanner()
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
22 : base(JSONGrammar.Instance.JsonDFA) {
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
23 m_stringBuffer = new char[1024];
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
24 var dfa = JSONGrammar.Instance.JsonStringDFA;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
25 m_stringAlphabet = dfa.Alphabet.GetTranslationMap();
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
26 m_stringDFA = dfa.States;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
27 }
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
28
60
10c7337d29e7 +JSONXmlReaderOptions
cin
parents: 55
diff changeset
29 /// <summary>
10c7337d29e7 +JSONXmlReaderOptions
cin
parents: 55
diff changeset
30 /// Читает следующий лексический элемент из входных данных.
10c7337d29e7 +JSONXmlReaderOptions
cin
parents: 55
diff changeset
31 /// </summary>
10c7337d29e7 +JSONXmlReaderOptions
cin
parents: 55
diff changeset
32 /// <param name="tokenValue">Возвращает значение прочитанного токена.</param>
10c7337d29e7 +JSONXmlReaderOptions
cin
parents: 55
diff changeset
33 /// <param name="tokenType">Возвращает тип прочитанного токена.</param>
10c7337d29e7 +JSONXmlReaderOptions
cin
parents: 55
diff changeset
34 /// <returns><c>true</c> - чтение произведено успешно. <c>false</c> - достигнут конец входных данных</returns>
10c7337d29e7 +JSONXmlReaderOptions
cin
parents: 55
diff changeset
35 /// <remarks>В случе если токен не распознается, возникает исключение. Значения токенов обрабатываются, т.е.
10c7337d29e7 +JSONXmlReaderOptions
cin
parents: 55
diff changeset
36 /// в строках обрабатываются экранированные символы, числа становтся типа double.</remarks>
55
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
37 public bool ReadToken(out object tokenValue, out JsonTokenType tokenType) {
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
38 if (ReadTokenInternal()) {
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
39 switch ((JSONGrammar.TokenType)m_currentState.tag[0]) {
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
40 case JSONGrammar.TokenType.StringBound:
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
41 tokenValue = ReadString();
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
42 tokenType = JsonTokenType.String;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
43 break;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
44 case JSONGrammar.TokenType.Number:
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
45 tokenValue = Double.Parse(new String(m_buffer, m_tokenOffset, m_tokenLen), CultureInfo.InvariantCulture);
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
46 tokenType = JsonTokenType.Number;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
47 break;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
48 default:
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
49 tokenType = (JsonTokenType)m_currentState.tag[0];
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
50 tokenValue = new String(m_buffer, m_tokenOffset, m_tokenLen);
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
51 break;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
52 }
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
53 return true;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
54 }
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
55 tokenValue = null;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
56 tokenType = JsonTokenType.None;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
57 return false;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
58 }
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
59
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
60 string ReadString() {
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
61 int pos = 0;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
62 Switch(m_stringDFA, m_stringAlphabet);
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
63 while (ReadTokenInternal()) {
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
64 switch ((JSONGrammar.TokenType)m_currentState.tag[0]) {
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
65 case JSONGrammar.TokenType.StringBound:
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
66 Restore();
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
67 return new String(m_stringBuffer, 0, pos);
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
68 case JSONGrammar.TokenType.UnescapedChar:
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
69 EnsureStringBufferSize(pos + m_tokenLen);
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
70 Array.Copy(m_buffer, m_tokenOffset, m_stringBuffer, pos, m_tokenLen);
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
71 pos += m_tokenLen;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
72 break;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
73 case JSONGrammar.TokenType.EscapedUnicode:
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
74 EnsureStringBufferSize(pos + 1);
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
75 m_stringBuffer[pos] = StringTranslator.TranslateHexUnicode(m_buffer, m_tokenOffset + 2);
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
76 pos++;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
77 break;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
78 case JSONGrammar.TokenType.EscapedChar:
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
79 EnsureStringBufferSize(pos + 1);
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
80 m_stringBuffer[pos] = StringTranslator.TranslateEscapedChar(m_buffer[m_tokenOffset + 1]);
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
81 pos++;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
82 break;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
83 default:
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
84 break;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
85 }
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
86
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
87 }
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
88
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
89 throw new ParserException("Unexpected end of data");
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
90 }
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
91
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
92 void EnsureStringBufferSize(int size) {
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
93 if (size > m_stringBuffer.Length) {
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
94 var newBuffer = new char[size];
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
95 m_stringBuffer.CopyTo(newBuffer, 0);
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
96 m_stringBuffer = newBuffer;
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
97 }
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
98 }
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
99 }
c0bf853aa04f Added initial JSON support
cin
parents:
diff changeset
100 }