Mercurial > pub > ImplabNet

changeset 180:c32688129f14 ref20160224

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
refactoring complete, JSONParser rewritten
author cin
date Thu, 24 Mar 2016 02:30:46 +0300
parents 478ef706906a
children b2b6a6640aa3
files .hgignore Implab/Automaton/DFATable.cs Implab/Automaton/DummyAlphabet.cs Implab/Automaton/EnumAlphabet.cs Implab/Automaton/IndexedAlphabetBase.cs Implab/Automaton/RegularExpressions/RegularDFA.cs Implab/Components/LazyAndWeak.cs Implab/Components/RunnableComponent.cs Implab/Formats/JSON/JSONElementContext.cs Implab/Formats/JSON/JSONGrammar.cs Implab/Formats/JSON/JSONParser.cs Implab/Formats/JSON/JSONScanner.cs Implab/Formats/JSON/JSONWriter.cs Implab/Formats/JSON/JSONXmlReader.cs Implab/Formats/JSON/JSONXmlReaderOptions.cs Implab/Formats/JSON/StringTranslator.cs Implab/Formats/TextScanner.cs Implab/Implab.csproj
diffstat 18 files changed, 126 insertions(+), 196 deletions(-) [+]
line wrap: on
line diff
--- a/.hgignore	Wed Mar 23 19:52:08 2016 +0300
+++ b/.hgignore	Thu Mar 24 02:30:46 2016 +0300
@@ -15,3 +15,5 @@
 Implab.Diagnostics.Interactive/obj/
 MonoPlay/bin/
 MonoPlay/obj/
+Implab.Test/Implab.Format.Test/bin/
+Implab.Test/Implab.Format.Test/obj/
--- a/Implab/Automaton/DFATable.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Automaton/DFATable.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -77,7 +77,7 @@
         }
 
         public bool Remove(AutomatonTransition item) {
-            m_transitions.Remove(item);
+            return m_transitions.Remove(item);
         }
 
         public int Count {
@@ -168,9 +168,9 @@
 
             var rmap = m_transitions
                 .GroupBy(t => t.s2)
-                .ToLookup(
+                .ToDictionary(
                     g => g.Key, // s2
-                    g => g.ToLookup(t => t.edge, t => t.s1)
+                    g => g.GroupBy(t => t.edge, t => t.s1).ToDictionary(p => p.Key)
                 );
 
             while (queue.Count > 0) {
@@ -180,7 +180,7 @@
                 for (int c = 0; c < m_symbolCount; c++) {
                     var stateX = new HashSet<int>();
                     foreach(var a in stateA)
-                        stateX.UnionWith(rmap[a][c]); // all states from wich 'c' leads to 'a'
+                        stateX.UnionWith(rmap[a][c]); // all states from wich the symbol 'c' leads to the state 'a'
 
                     foreach (var stateY in optimalStates.ToArray()) {
                         if (stateX.Overlaps(stateY) && !stateY.IsSubsetOf(stateX)) {
--- a/Implab/Automaton/DummyAlphabet.cs	Wed Mar 23 19:52:08 2016 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,46 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-namespace Implab.Automaton {
-    /// <summary>
-    /// Dummy alphabet consists of integer numbers which are identical to their classes.
-    /// </summary>
-    public class DummyAlphabet : IAlphabet<int> {
-        readonly int m_size;
-
-        /// <summary>
-        /// Creates a new dummy alphabet with given size.
-        /// </summary>
-        /// <param name="size">The size of the alphabet, must be greater then zero.</param>
-        public DummyAlphabet(int size) {
-            Safe.ArgumentAssert(size > 0);
-            m_size = 0;
-        }
-
-        #region IAlphabet implementation
-
-        public List<int>[] CreateReverseMap() {
-            Enumerable.Range(0, m_size).ToArray();
-        }
-
-        public int Translate(int symbol) {
-            Safe.ArgumentInRange(symbol, 0, m_size, "symbol");
-            return symbol;
-        }
-
-        public bool Contains(int symbol) {
-            Safe.ArgumentInRange(symbol, 0, m_size, "symbol");
-            return true;
-        }
-
-        public int Count {
-            get {
-                return m_size;
-            }
-        }
-
-        #endregion
-    }
-}
-
--- a/Implab/Automaton/EnumAlphabet.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Automaton/EnumAlphabet.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -62,9 +62,5 @@
             return symbol.ToInt32(CultureInfo.InvariantCulture);
         }
 
-        public override IEnumerable<T> InputSymbols {
-            get { return _symbols.Value; }
-        }
-
     }
 }
--- a/Implab/Automaton/IndexedAlphabetBase.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Automaton/IndexedAlphabetBase.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -30,9 +30,9 @@
         /// </remarks>
         /// <returns>The translation map.</returns>
         public int[] GetTranslationMap() {
-            Dictionary<int,int> map = new Dictionary<int, int>();
+            var map = new Dictionary<int, int>();
 
-            int max;
+            int max = 0;
             foreach (var p in Mappings) {
                 var index = GetSymbolIndex(p.Key);
                 max = Math.Max(max, index);
--- a/Implab/Automaton/RegularExpressions/RegularDFA.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Automaton/RegularExpressions/RegularDFA.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -53,7 +53,6 @@
 
             var dfa = new RegularDFA<TInput, TTag>(alphabet);
 
-            var states = new DummyAlphabet(StateCount);
             var alphaMap = new Dictionary<int,int>();
             var stateMap = new Dictionary<int,int>();
 
--- a/Implab/Components/LazyAndWeak.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Components/LazyAndWeak.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -2,6 +2,13 @@
 using System.Threading;
 
 namespace Implab.Components {
+    /// <summary>
+    /// Creates an instace on-demand and allows it to be garbage collected.
+    /// </summary>
+    /// <remarks>
+    /// Usefull when dealing with memory-intensive objects which are frequently used.
+    /// This class is similar to <see cref="ObjectPool{T}"/> except is a singleton.
+    /// </remarks>
     public class LazyAndWeak<T> where T : class {
 
         readonly Func<T> m_factory;
@@ -35,6 +42,18 @@
                         if (Interlocked.CompareExchange(ref m_reference, new WeakReference(value), weak) == weak)
                             return value;
                     } else {
+                        lock (m_lock) {
+                            // double check
+                            if (weak != null) {
+                                value = weak.Target as T;
+                                if (value != null)
+                                    return value;
+                            }
+                            // we are safe to write
+                            value = m_factory();
+                            m_reference = new WeakReference(value);
+                            return value;
+                        }
                     }
                 }
             }
--- a/Implab/Components/RunnableComponent.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Components/RunnableComponent.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -1,5 +1,5 @@
 using System;
-using Implab.Parsing;
+using Implab.Formats;
 
 namespace Implab.Components {
     public class RunnableComponent : Disposable, IRunnable, IInitializable {
--- a/Implab/Formats/JSON/JSONElementContext.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Formats/JSON/JSONElementContext.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -5,6 +5,7 @@
     enum JSONElementContext {
         None,
         Object,
-        Array
+        Array,
+        Closed
     }
 }
--- a/Implab/Formats/JSON/JSONGrammar.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Formats/JSON/JSONGrammar.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -2,6 +2,7 @@
 using Implab.Automaton.RegularExpressions;
 using System;
 using Implab.Automaton;
+using Implab.Components;
 
 namespace Implab.Formats.JSON {
     class JSONGrammar : Grammar<char> {
@@ -23,7 +24,7 @@
             EscapedUnicode
         }
 
-        static Lazy<JSONGrammar> _instance = new Lazy<JSONGrammar>();
+        static LazyAndWeak<JSONGrammar> _instance = new LazyAndWeak<JSONGrammar>(() => new JSONGrammar());
 
         public static JSONGrammar Instance {
             get { return _instance.Value; }
@@ -31,6 +32,7 @@
 
         readonly ScannerContext<TokenType> m_jsonExpression;
         readonly ScannerContext<TokenType> m_stringExpression;
+        readonly CharAlphabet m_defaultAlphabet = new CharAlphabet();
 
         public JSONGrammar() {
             DefineAlphabet(Enumerable.Range(0, 0x20).Select(x => (char)x));
@@ -87,6 +89,12 @@
 
         }
 
+        protected override IAlphabetBuilder<char> AlphabetBuilder {
+            get {
+                return m_defaultAlphabet;
+            }
+        }
+
         public ScannerContext<TokenType> JsonExpression {
             get {
                 return m_jsonExpression;
@@ -103,7 +111,7 @@
             return SymbolToken(Enumerable.Range(start,stop - start).Cast<char>());
         }
 
-        protected override IAlphabetBuilder<char> CreateAlphabet() {
+        protected override IndexedAlphabetBase<char> CreateAlphabet() {
             return new CharAlphabet();
         }
                 
--- a/Implab/Formats/JSON/JSONParser.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Formats/JSON/JSONParser.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -5,17 +5,10 @@
 using Implab.Automaton.RegularExpressions;
 using System.Linq;
 using Implab.Components;
+using System.Collections.Generic;
 
 namespace Implab.Formats.JSON {
     /// <summary>
-    /// internal
-    /// </summary>
-    public struct JSONParserContext {
-        public string memberName;
-        public JSONElementContext elementContext;
-    }
-
-    /// <summary>
     /// Pull парсер JSON данных.
     /// </summary>
     /// <remarks>
@@ -52,10 +45,11 @@
             }
 
             public bool Move(JsonTokenType token) {
-                var next = m_dfa[m_state, token];
+                var next = m_dfa[m_state, (int)token];
                 if (next == AutomatonConst.UNREACHABLE_STATE)
                     return false;
                 m_state = next;
+                return true;
             }
 
             public JSONElementContext ElementContext {
@@ -63,40 +57,43 @@
             }
         }
 
+        static readonly ParserContext _jsonContext;
+        static readonly ParserContext _objectContext;
+        static readonly ParserContext _arrayContext;
+
         static JSONParser() {
 
-
-            var valueExpression = Token(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
-            var memberExpression = Token(JsonTokenType.String).Cat(Token(JsonTokenType.NameSeparator)).Cat(valueExpression);
+            var valueExpression = MakeToken(JsonTokenType.BeginArray, JsonTokenType.BeginObject, JsonTokenType.Literal, JsonTokenType.Number, JsonTokenType.String);
+            var memberExpression = MakeToken(JsonTokenType.String).Cat(MakeToken(JsonTokenType.NameSeparator)).Cat(valueExpression);
 
             var objectExpression = memberExpression
                 .Cat(
-                    Token(JsonTokenType.ValueSeparator)
+                    MakeToken(JsonTokenType.ValueSeparator)
                     .Cat(memberExpression)
                     .EClosure()
                 )
                 .Optional()
-                .Cat(Token(JsonTokenType.EndObject))
+                .Cat(MakeToken(JsonTokenType.EndObject))
                 .End();
             
             var arrayExpression = valueExpression
                 .Cat(
-                    Token(JsonTokenType.ValueSeparator)
+                    MakeToken(JsonTokenType.ValueSeparator)
                     .Cat(valueExpression)
                     .EClosure()
                 )
                 .Optional()
-                .Cat(Token(JsonTokenType.EndArray))
+                .Cat(MakeToken(JsonTokenType.EndArray))
                 .End();
 
             var jsonExpression = valueExpression.End();
 
-            _jsonDFA = CreateParserContext(jsonExpression, JSONElementContext.None);
-            _objectDFA = CreateParserContext(objectExpression, JSONElementContext.Object);
-            _arrayDFA = CreateParserContext(arrayExpression, JSONElementContext.Array);
+            _jsonContext = CreateParserContext(jsonExpression, JSONElementContext.None);
+            _objectContext = CreateParserContext(objectExpression, JSONElementContext.Object);
+            _arrayContext = CreateParserContext(arrayExpression, JSONElementContext.Array);
         }
 
-        static Token Token(params JsonTokenType[] input) {
+        static Token MakeToken(params JsonTokenType[] input) {
             return Token.New( input.Select(t => (int)t).ToArray() );
         }
 
@@ -112,32 +109,36 @@
 
         #endregion
 
-        JSONScanner m_scanner;
+        readonly JSONScanner m_scanner;
         MemberContext m_memberContext;
 
         JSONElementType m_elementType;
         object m_elementValue;
+        string m_memberName = String.Empty;
+
+        Stack<ParserContext> m_stack = new Stack<ParserContext>();
+        ParserContext m_context = _jsonContext;
 
         /// <summary>
         /// Создает новый парсер на основе строки, содержащей JSON
         /// </summary>
         /// <param name="text"></param>
-        public JSONParser(string text)
-            : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) {
+        public JSONParser(string text) {
             Safe.ArgumentNotEmpty(text, "text");
-            m_scanner = new JSONScanner();
-            m_scanner.Feed(text.ToCharArray());
+            m_scanner = new JSONScanner(text);
         }
 
         /// <summary>
         /// Создает новый экземпляр парсера, на основе текстового потока.
         /// </summary>
         /// <param name="reader">Текстовый поток.</param>
-        public JSONParser(TextReader reader)
-            : base(_jsonDFA, INITIAL_STATE, new JSONParserContext { elementContext = JSONElementContext.None, memberName = String.Empty }) {
+        public JSONParser(TextReader reader) {
             Safe.ArgumentNotNull(reader, "reader");
-            m_scanner = new JSONScanner();
-            m_scanner.Feed(reader, dispose);
+            m_scanner = new JSONScanner(reader);
+        }
+
+        public int Level {
+            get { return m_stack.Count; }
         }
 
         /// <summary>
@@ -152,7 +153,7 @@
         /// пустая строка.
         /// </summary>
         public string ElementName {
-            get { return m_context.info.memberName; }
+            get { return m_memberName; }
         }
 
         /// <summary>
@@ -167,55 +168,51 @@
         /// </summary>
         /// <returns><c>true</c> - операция чтения прошла успешно, <c>false</c> - конец данных</returns>
         public bool Read() {
-            if (m_context.current == UNREACHEBLE_STATE)
-                throw new InvalidOperationException("The parser is in invalid state");
             object tokenValue;
             JsonTokenType tokenType;
-            m_context.info.memberName = String.Empty;
+
+            m_memberName = String.Empty;
+
             while (m_scanner.ReadToken(out tokenValue, out tokenType)) {
-                Move((int)tokenType);
-                if (m_context.current == UNREACHEBLE_STATE)
+                if(!m_context.Move(tokenType))
                     UnexpectedToken(tokenValue, tokenType);
+
                 switch (tokenType) {
                     case JsonTokenType.BeginObject:
-                        Switch(
-                            _objectDFA,
-                            INITIAL_STATE,
-                            new JSONParserContext {
-                                memberName = m_context.info.memberName,
-                                elementContext = JSONElementContext.Object
-                            }
-                        );
+                        m_stack.Push(m_context);
+                        m_context = _objectContext;
+
                         m_elementValue = null;
                         m_memberContext = MemberContext.MemberName;
                         m_elementType = JSONElementType.BeginObject;
                         return true;
                     case JsonTokenType.EndObject:
-                        Restore();
+                        if (m_stack.Count == 0)
+                            UnexpectedToken(tokenValue, tokenType);
+                        m_context = m_stack.Pop();
+
                         m_elementValue = null;
                         m_elementType = JSONElementType.EndObject;
                         return true;
                     case JsonTokenType.BeginArray:
-                        Switch(
-                            _arrayDFA,
-                            INITIAL_STATE,
-                            new JSONParserContext {
-                                memberName = m_context.info.memberName,
-                                elementContext = JSONElementContext.Array
-                            }
-                        );
+                        m_stack.Push(m_context);
+                        m_context = _arrayContext;
+
                         m_elementValue = null;
                         m_memberContext = MemberContext.MemberValue;
                         m_elementType = JSONElementType.BeginArray;
                         return true;
                     case JsonTokenType.EndArray:
-                        Restore();
+                        if (m_stack.Count == 0)
+                            UnexpectedToken(tokenValue, tokenType);
+                        m_context = m_stack.Pop();
+
                         m_elementValue = null;
                         m_elementType = JSONElementType.EndArray;
                         return true;
                     case JsonTokenType.String:
                         if (m_memberContext == MemberContext.MemberName) {
-                            m_context.info.memberName = (string)tokenValue;
+                            m_memberName = (string)tokenValue;
                             break;
                         }
                         m_elementType = JSONElementType.Value;
@@ -233,15 +230,18 @@
                         m_memberContext = MemberContext.MemberValue;
                         break;
                     case JsonTokenType.ValueSeparator:
-                        m_memberContext = m_context.info.elementContext == JSONElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
+                        m_memberContext = m_context.ElementContext == JSONElementContext.Object ? MemberContext.MemberName : MemberContext.MemberValue;
                         break;
                     default:
                         UnexpectedToken(tokenValue, tokenType);
                         break;
                 }
             }
-            if (m_context.info.elementContext != JSONElementContext.None)
+            if (m_context.ElementContext != JSONElementContext.None)
                 throw new ParserException("Unexpedted end of data");
+
+            EOF = true;
+
             return false;
         }
 
@@ -268,15 +268,13 @@
         /// Признак конца потока
         /// </summary>
         public bool EOF {
-            get {
-                return m_scanner.EOF;
-            }
+            get;
+            private set;
         }
 
         protected override void Dispose(bool disposing) {
-            if (disposing) {
-                m_scanner.Dispose();
-            }
+            if (disposing)
+                Safe.Dispose(m_scanner);
         }
 
         /// <summary>
--- a/Implab/Formats/JSON/JSONScanner.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Formats/JSON/JSONScanner.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -33,6 +33,9 @@
             m_scanner = new ReaderScanner(reader, bufferMax, chunkSize);
         }
 
+        public JSONScanner(TextReader reader) : this(reader, 1024*1024, 1024){
+        }
+
         /// <summary>
         /// Читает следующий лексический элемент из входных данных.
         /// </summary>
--- a/Implab/Formats/JSON/JSONWriter.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Formats/JSON/JSONWriter.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -4,7 +4,7 @@
 using System.Globalization;
 using System.Diagnostics;
 
-namespace Implab.JSON {
+namespace Implab.Formats.JSON {
     public class JSONWriter {
         struct Context {
             public bool needComma;
--- a/Implab/Formats/JSON/JSONXmlReader.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Formats/JSON/JSONXmlReader.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -1,15 +1,11 @@
 using Implab;
-using Implab.Parsing;
 using System;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using System.Xml;
 
-namespace Implab.JSON {
+namespace Implab.Formats.JSON {
     public class JSONXmlReader : XmlReader {
 
         enum ValueContext {
@@ -30,7 +26,7 @@
         ReadState m_state = ReadState.Initial;
         Stack<LocalNameContext> m_localNameStack = new Stack<LocalNameContext>();
         LocalNameContext m_localName;
-        int m_depthCorrection = 0;
+        int m_depthCorrection;
 
         readonly string m_rootName;
         readonly string m_prefix;
@@ -119,8 +115,8 @@
         public override string LookupNamespace(string prefix) {
             if (String.IsNullOrEmpty(prefix) || prefix == m_prefix)
                 return m_namespaceUri;
-            else
-                return String.Empty;
+            
+            return String.Empty;
         }
 
         public override bool MoveToAttribute(string name, string ns) {
@@ -183,11 +179,11 @@
         }
 
         public override bool Read() {
-            if (m_state != System.Xml.ReadState.Interactive && m_state != System.Xml.ReadState.Initial)
+            if (m_state != ReadState.Interactive && m_state != ReadState.Initial)
                 return false;
 
             if (m_state == ReadState.Initial)
-                m_state = System.Xml.ReadState.Interactive;
+                m_state = ReadState.Interactive;
 
             try {
                 switch (m_parser.ElementType) {
@@ -223,9 +219,8 @@
                                 m_depthCorrection--;
                                 SetLocalName(itemName, true);
                                 continue;
-                            } else {
-                                SetLocalName(itemName, true);
                             }
+                            SetLocalName(itemName, true);
                             break;
                         case JSONElementType.BeginObject:
                             SetLocalName(itemName);
@@ -243,16 +238,14 @@
                             SetLocalName(itemName);
                             m_valueContext = m_parser.ElementValue == null ? ValueContext.ElementEmpty : ValueContext.ElementStart;
                             break;
-                        default:
-                            break;
                     }
                     return true;
                 }
 
-                m_state = System.Xml.ReadState.EndOfFile;
+                m_state = ReadState.EndOfFile;
                 return false;
             } catch {
-                m_state = System.Xml.ReadState.Error;
+                m_state = ReadState.Error;
                 throw;
             }
         }
@@ -275,8 +268,7 @@
                     return String.Empty;
                 if (Convert.GetTypeCode(m_parser.ElementValue) == TypeCode.Double)
                     return ((double)m_parser.ElementValue).ToString(CultureInfo.InvariantCulture);
-                else
-                    return m_parser.ElementValue.ToString();
+                return m_parser.ElementValue.ToString();
             }
         }
 
@@ -323,7 +315,7 @@
         /// The reader will be disposed when the XmlReader is disposed.
         /// </remarks>
         public static JSONXmlReader Create(TextReader reader,  JSONXmlReaderOptions options) {
-            return new JSONXmlReader(new JSONParser(reader, true), options);
+            return new JSONXmlReader(new JSONParser(reader), options);
         }
 
         /// <summary>
--- a/Implab/Formats/JSON/JSONXmlReaderOptions.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Formats/JSON/JSONXmlReaderOptions.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -1,10 +1,7 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+
 using System.Xml;
 
-namespace Implab.JSON {
+namespace Implab.Formats.JSON {
     /// <summary>
     /// Набор необязательных параметров для <see cref="JSONXmlReader"/>, позволяющий управлять процессом
     /// интерпретации <c>JSON</c> документа.
--- a/Implab/Formats/JSON/StringTranslator.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Formats/JSON/StringTranslator.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -11,7 +11,7 @@
     /// <summary>
     /// Класс для преобразования экранированной строки JSON
     /// </summary>
-    public class StringTranslator : TextScanner<JSONGrammar.TokenType> {
+    static class StringTranslator {
         static readonly char[] _escMap;
         static readonly int[] _hexMap;
 
@@ -34,49 +34,6 @@
 
         }
 
-        public StringTranslator() {
-        }
-
-        public string Translate(string data) {
-            Safe.ArgumentNotNull(data, "data");
-            return Translate(data.ToCharArray());
-        }
-
-        public string Translate(char[] data) {
-            Safe.ArgumentNotNull(data, "data");
-            return Translate(data, data.Length);
-        }
-
-        public string Translate(char[] data, int length) {
-            Safe.ArgumentNotNull(data, "data");
-            Safe.ArgumentInRange(length, 0, data.Length, "length");
-
-            var translated = new char[length];
-
-            Feed(data,length);
-
-            int pos = 0;
-
-            while (ReadTokenInternal()) {
-                switch ((JSONGrammar.TokenType)Tags[0]) {
-                    case JSONGrammar.TokenType.UnescapedChar:
-                        Array.Copy(m_buffer,m_tokenOffset,translated,pos,m_tokenLen);
-                        pos += m_tokenLen;
-                        break;
-                    case JSONGrammar.TokenType.EscapedChar:
-                        translated[pos] = _escMap[m_buffer[m_tokenOffset + 1]];
-                        pos++;
-                        break;
-                    case JSONGrammar.TokenType.EscapedUnicode:
-                        translated[pos] = TranslateHexUnicode(m_buffer,m_tokenOffset + 2);
-                        pos++;
-                        break;
-                }
-            }
-
-            return new String(translated, 0, pos);
-        }
-
         internal static char TranslateEscapedChar(char symbol) {
             return _escMap[symbol];
         }
--- a/Implab/Formats/TextScanner.cs	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Formats/TextScanner.cs	Thu Mar 24 02:30:46 2016 +0300
@@ -49,8 +49,8 @@
         /// <param name="alphabet"></param>
         /// <param name = "tag"></param>
         internal bool ReadToken<TTag>(int[,] dfa, bool[] final, TTag[][] tags, int state, int[] alphabet, out TTag[] tag) {
-            Safe.ArgumentNotNull();
             m_tokenLength = 0;
+            tag = null;
 
             var maxSymbol = alphabet.Length - 1;
 
@@ -109,7 +109,7 @@
                     var size = used + free;
 
                     if (size > m_bufferMax)
-                        throw new ParserException(String.Format("The buffer limit ({0} Kb) is reached", m_bufferMax/1024));
+                        throw new ParserException(String.Format("The buffer limit ({0} Kb) is reached", m_bufferMax / 1024));
                     
                     var temp = new char[size];
 
@@ -122,7 +122,13 @@
                     m_bufferOffset = 0;
                     m_bufferSize = used + read;
                     m_buffer = temp;
+                } else {
+                    var read = Read(m_buffer, m_bufferSize, m_chunkSize);
+                    if (read == 0)
+                        return false;
+                    m_bufferSize += m_chunkSize;
                 }
+                return true;
             } else {
                 Debug.Assert(m_bufferOffset == 0);
                 m_buffer = new char[m_chunkSize];
--- a/Implab/Implab.csproj	Wed Mar 23 19:52:08 2016 +0300
+++ b/Implab/Implab.csproj	Thu Mar 24 02:30:46 2016 +0300
@@ -151,7 +151,6 @@
     <Compile Include="Components\ExecutionState.cs" />
     <Compile Include="Components\RunnableComponent.cs" />
     <Compile Include="Components\IFactory.cs" />
-    <Compile Include="Automaton\EnumAlphabet.cs" />
     <Compile Include="Automaton\IAlphabet.cs" />
     <Compile Include="Automaton\ParserException.cs" />
     <Compile Include="Automaton\IndexedAlphabetBase.cs" />
@@ -176,7 +175,6 @@
     <Compile Include="Formats\JSON\JSONXmlReaderOptions.cs" />
     <Compile Include="Formats\JSON\StringTranslator.cs" />
     <Compile Include="Automaton\MapAlphabet.cs" />
-    <Compile Include="Automaton\DummyAlphabet.cs" />
     <Compile Include="Formats\CharAlphabet.cs" />
     <Compile Include="Formats\ByteAlphabet.cs" />
     <Compile Include="Automaton\IDFATable.cs" />