lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nightowl...@apache.org
Subject [43/62] [abbrv] lucenenet git commit: Lucene.Net.Core.Analysis: Deleted obsolete Analysis files that have mostly been moved to Lucene.Net.Analysis.Common
Date Sat, 01 Apr 2017 01:09:36 GMT
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8a97bfcf/src/Lucene.Net.Core/Analysis/PorterStemmer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Analysis/PorterStemmer.cs b/src/Lucene.Net.Core/Analysis/PorterStemmer.cs
deleted file mode 100644
index bc4cf75..0000000
--- a/src/Lucene.Net.Core/Analysis/PorterStemmer.cs
+++ /dev/null
@@ -1,746 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
-
-Porter stemmer in Java. The original paper is in
-
-Porter, 1980, An algorithm for suffix stripping, Program, Vol. 14,
-no. 3, pp 130-137,
-
-See also http://www.tartarus.org/~martin/PorterStemmer/index.html
-
-Bug 1 (reported by Gonzalo Parra 16/10/99) fixed as marked below.
-Tthe words 'aed', 'eed', 'oed' leave k at 'a' for step 3, and b[k-1]
-is then out outside the bounds of b.
-
-Similarly,
-
-Bug 2 (reported by Steve Dyrdahl 22/2/00) fixed as marked below.
-'ion' by itself leaves j = -1 in the test for 'ion' in step 5, and
-b[j] is then outside the bounds of b.
-
-Release 3.
-
-[ This version is derived from Release 3, modified by Brian Goetz to
-optimize for fewer object creations.  ]
-*/
-using System;
-namespace Lucene.Net.Analysis
-{
-    
-    /// <summary> 
-    /// Stemmer, implementing the Porter Stemming Algorithm
-    /// 
-    /// The Stemmer class transforms a word into its root form.  The input
-    /// word can be provided a character at time (by calling add()), or at once
-    /// by calling one of the various stem(something) methods.
-    /// </summary>
-    
-    class PorterStemmer
-    {
-        private char[] b;
-        private int i, j, k, k0;
-        private bool dirty = false;
-        private const int INC = 50; /* unit of size whereby b is increased */
-        private const int EXTRA = 1;
-        
-        public PorterStemmer()
-        {
-            b = new char[INC];
-            i = 0;
-        }
-        
-        /// <summary> reset() resets the stemmer so it can stem another word.  If you invoke
-        /// the stemmer by calling add(char) and then stem(), you must call reset()
-        /// before starting another word.
-        /// </summary>
-        public virtual void  Reset()
-        {
-            i = 0; dirty = false;
-        }
-        
-        /// <summary> Add a character to the word being stemmed.  When you are finished
-        /// adding characters, you can call stem(void) to process the word.
-        /// </summary>
-        public virtual void  Add(char ch)
-        {
-            if (b.Length <= i + EXTRA)
-            {
-                var new_b = new char[b.Length + INC];
-                Array.Copy(b, 0, new_b, 0, b.Length);
-                b = new_b;
-            }
-            b[i++] = ch;
-        }
-        
-        /// <summary> After a word has been stemmed, it can be retrieved by toString(),
-        /// or a reference to the internal buffer can be retrieved by getResultBuffer
-        /// and getResultLength (which is generally more efficient.)
-        /// </summary>
-        public override System.String ToString()
-        {
-            return new System.String(b, 0, i);
-        }
-
-        /// <summary> Returns the length of the word resulting from the stemming process.</summary>
-        public virtual int ResultLength
-        {
-            get { return i; }
-        }
-
-        /// <summary> Returns a reference to a character buffer containing the results of
-        /// the stemming process.  You also need to consult getResultLength()
-        /// to determine the length of the result.
-        /// </summary>
-        public virtual char[] ResultBuffer
-        {
-            get { return b; }
-        }
-
-        /* cons(i) is true <=> b[i] is a consonant. */
-        
-        private bool Cons(int i)
-        {
-            switch (b[i])
-            {
-                
-                case 'a': 
-                case 'e': 
-                case 'i': 
-                case 'o': 
-                case 'u': 
-                    return false;
-                
-                case 'y': 
-                    return (i == k0)?true:!Cons(i - 1);
-                
-                default: 
-                    return true;
-                
-            }
-        }
-        
-        /* m() measures the number of consonant sequences between k0 and j. if c is
-        a consonant sequence and v a vowel sequence, and <..> indicates arbitrary
-        presence,
-        
-        <c><v>       gives 0
-        <c>vc<v>     gives 1
-        <c>vcvc<v>   gives 2
-        <c>vcvcvc<v> gives 3
-        ....
-        */
-        
-        private int M()
-        {
-            int n = 0;
-            int i = k0;
-            while (true)
-            {
-                if (i > j)
-                    return n;
-                if (!Cons(i))
-                    break;
-                i++;
-            }
-            i++;
-            while (true)
-            {
-                while (true)
-                {
-                    if (i > j)
-                        return n;
-                    if (Cons(i))
-                        break;
-                    i++;
-                }
-                i++;
-                n++;
-                while (true)
-                {
-                    if (i > j)
-                        return n;
-                    if (!Cons(i))
-                        break;
-                    i++;
-                }
-                i++;
-            }
-        }
-        
-        /* vowelinstem() is true <=> k0,...j contains a vowel */
-        
-        private bool Vowelinstem()
-        {
-            int i;
-            for (i = k0; i <= j; i++)
-                if (!Cons(i))
-                    return true;
-            return false;
-        }
-        
-        /* doublec(j) is true <=> j,(j-1) contain a double consonant. */
-        
-        private bool Doublec(int j)
-        {
-            if (j < k0 + 1)
-                return false;
-            if (b[j] != b[j - 1])
-                return false;
-            return Cons(j);
-        }
-        
-        /* cvc(i) is true <=> i-2,i-1,i has the form consonant - vowel - consonant
-        and also if the second c is not w,x or y. this is used when trying to
-        restore an e at the end of a short word. e.g.
-        
-        cav(e), lov(e), hop(e), crim(e), but
-        snow, box, tray.
-        
-        */
-        
-        private bool Cvc(int i)
-        {
-            if (i < k0 + 2 || !Cons(i) || Cons(i - 1) || !Cons(i - 2))
-                return false;
-            else
-            {
-                int ch = b[i];
-                if (ch == 'w' || ch == 'x' || ch == 'y')
-                    return false;
-            }
-            return true;
-        }
-        
-        private bool Ends(System.String s)
-        {
-            int l = s.Length;
-            int o = k - l + 1;
-            if (o < k0)
-                return false;
-            for (int i = 0; i < l; i++)
-                if (b[o + i] != s[i])
-                    return false;
-            j = k - l;
-            return true;
-        }
-        
-        /* setto(s) sets (j+1),...k to the characters in the string s, readjusting
-        k. */
-        
-        internal virtual void  Setto(System.String s)
-        {
-            int l = s.Length;
-            int o = j + 1;
-            for (int i = 0; i < l; i++)
-                b[o + i] = s[i];
-            k = j + l;
-            dirty = true;
-        }
-        
-        /* r(s) is used further down. */
-        
-        internal virtual void  R(System.String s)
-        {
-            if (M() > 0)
-                Setto(s);
-        }
-        
-        /* step1() gets rid of plurals and -ed or -ing. e.g.
-        
-        caresses  ->  caress
-        ponies    ->  poni
-        ties      ->  ti
-        caress    ->  caress
-        cats      ->  cat
-        
-        feed      ->  feed
-        agreed    ->  agree
-        disabled  ->  disable
-        
-        matting   ->  mat
-        mating    ->  mate
-        meeting   ->  meet
-        milling   ->  mill
-        messing   ->  mess
-        
-        meetings  ->  meet
-        
-        */
-        
-        private void  Step1()
-        {
-            if (b[k] == 's')
-            {
-                if (Ends("sses"))
-                    k -= 2;
-                else if (Ends("ies"))
-                    Setto("i");
-                else if (b[k - 1] != 's')
-                    k--;
-            }
-            if (Ends("eed"))
-            {
-                if (M() > 0)
-                    k--;
-            }
-            else if ((Ends("ed") || Ends("ing")) && Vowelinstem())
-            {
-                k = j;
-                if (Ends("at"))
-                    Setto("ate");
-                else if (Ends("bl"))
-                    Setto("ble");
-                else if (Ends("iz"))
-                    Setto("ize");
-                else if (Doublec(k))
-                {
-                    int ch = b[k--];
-                    if (ch == 'l' || ch == 's' || ch == 'z')
-                        k++;
-                }
-                else if (M() == 1 && Cvc(k))
-                    Setto("e");
-            }
-        }
-        
-        /* step2() turns terminal y to i when there is another vowel in the stem. */
-        
-        private void  Step2()
-        {
-            if (Ends("y") && Vowelinstem())
-            {
-                b[k] = 'i';
-                dirty = true;
-            }
-        }
-        
-        /* step3() maps double suffices to single ones. so -ization ( = -ize plus
-        -ation) maps to -ize etc. note that the string before the suffix must give
-        m() > 0. */
-        
-        private void  Step3()
-        {
-            if (k == k0)
-                return ; /* For Bug 1 */
-            switch (b[k - 1])
-            {
-                
-                case 'a': 
-                    if (Ends("ational"))
-                    {
-                        R("ate"); break;
-                    }
-                    if (Ends("tional"))
-                    {
-                        R("tion"); break;
-                    }
-                    break;
-                
-                case 'c': 
-                    if (Ends("enci"))
-                    {
-                        R("ence"); break;
-                    }
-                    if (Ends("anci"))
-                    {
-                        R("ance"); break;
-                    }
-                    break;
-                
-                case 'e': 
-                    if (Ends("izer"))
-                    {
-                        R("ize"); break;
-                    }
-                    break;
-                
-                case 'l': 
-                    if (Ends("bli"))
-                    {
-                        R("ble"); break;
-                    }
-                    if (Ends("alli"))
-                    {
-                        R("al"); break;
-                    }
-                    if (Ends("entli"))
-                    {
-                        R("ent"); break;
-                    }
-                    if (Ends("eli"))
-                    {
-                        R("e"); break;
-                    }
-                    if (Ends("ousli"))
-                    {
-                        R("ous"); break;
-                    }
-                    break;
-                
-                case 'o': 
-                    if (Ends("ization"))
-                    {
-                        R("ize"); break;
-                    }
-                    if (Ends("ation"))
-                    {
-                        R("ate"); break;
-                    }
-                    if (Ends("ator"))
-                    {
-                        R("ate"); break;
-                    }
-                    break;
-                
-                case 's': 
-                    if (Ends("alism"))
-                    {
-                        R("al"); break;
-                    }
-                    if (Ends("iveness"))
-                    {
-                        R("ive"); break;
-                    }
-                    if (Ends("fulness"))
-                    {
-                        R("ful"); break;
-                    }
-                    if (Ends("ousness"))
-                    {
-                        R("ous"); break;
-                    }
-                    break;
-                
-                case 't': 
-                    if (Ends("aliti"))
-                    {
-                        R("al"); break;
-                    }
-                    if (Ends("iviti"))
-                    {
-                        R("ive"); break;
-                    }
-                    if (Ends("biliti"))
-                    {
-                        R("ble"); break;
-                    }
-                    break;
-                
-                case 'g': 
-                    if (Ends("logi"))
-                    {
-                        R("log"); break;
-                    }
-                    break;
-                }
-        }
-        
-        /* step4() deals with -ic-, -full, -ness etc. similar strategy to step3. */
-        
-        private void  Step4()
-        {
-            switch (b[k])
-            {
-                
-                case 'e': 
-                    if (Ends("icate"))
-                    {
-                        R("ic"); break;
-                    }
-                    if (Ends("ative"))
-                    {
-                        R(""); break;
-                    }
-                    if (Ends("alize"))
-                    {
-                        R("al"); break;
-                    }
-                    break;
-                
-                case 'i': 
-                    if (Ends("iciti"))
-                    {
-                        R("ic"); break;
-                    }
-                    break;
-                
-                case 'l': 
-                    if (Ends("ical"))
-                    {
-                        R("ic"); break;
-                    }
-                    if (Ends("ful"))
-                    {
-                        R(""); break;
-                    }
-                    break;
-                
-                case 's': 
-                    if (Ends("ness"))
-                    {
-                        R(""); break;
-                    }
-                    break;
-                }
-        }
-        
-        /* step5() takes off -ant, -ence etc., in context <c>vcvc<v>. */
-        
-        private void  Step5()
-        {
-            if (k == k0)
-                return ; /* for Bug 1 */
-            switch (b[k - 1])
-            {
-                
-                case 'a': 
-                    if (Ends("al"))
-                        break;
-                    return ;
-                
-                case 'c': 
-                    if (Ends("ance"))
-                        break;
-                    if (Ends("ence"))
-                        break;
-                    return ;
-                
-                case 'e': 
-                    if (Ends("er"))
-                        break; return ;
-                
-                case 'i': 
-                    if (Ends("ic"))
-                        break; return ;
-                
-                case 'l': 
-                    if (Ends("able"))
-                        break;
-                    if (Ends("ible"))
-                        break; return ;
-                
-                case 'n': 
-                    if (Ends("ant"))
-                        break;
-                    if (Ends("ement"))
-                        break;
-                    if (Ends("ment"))
-                        break;
-                    /* element etc. not stripped before the m */
-                    if (Ends("ent"))
-                        break;
-                    return ;
-                
-                case 'o': 
-                    if (Ends("ion") && j >= 0 && (b[j] == 's' || b[j] == 't'))
-                        break;
-                    /* j >= 0 fixes Bug 2 */
-                    if (Ends("ou"))
-                        break;
-                    return ;
-                    /* takes care of -ous */
-                
-                case 's': 
-                    if (Ends("ism"))
-                        break;
-                    return ;
-                
-                case 't': 
-                    if (Ends("ate"))
-                        break;
-                    if (Ends("iti"))
-                        break;
-                    return ;
-                
-                case 'u': 
-                    if (Ends("ous"))
-                        break;
-                    return ;
-                
-                case 'v': 
-                    if (Ends("ive"))
-                        break;
-                    return ;
-                
-                case 'z': 
-                    if (Ends("ize"))
-                        break;
-                    return ;
-                
-                default: 
-                    return ;
-                
-            }
-            if (M() > 1)
-                k = j;
-        }
-        
-        /* step6() removes a final -e if m() > 1. */
-        
-        private void  Step6()
-        {
-            j = k;
-            if (b[k] == 'e')
-            {
-                int a = M();
-                if (a > 1 || a == 1 && !Cvc(k - 1))
-                    k--;
-            }
-            if (b[k] == 'l' && Doublec(k) && M() > 1)
-                k--;
-        }
-        
-        
-        /// <summary> Stem a word provided as a String.  Returns the result as a String.</summary>
-        public virtual System.String Stem(System.String s)
-        {
-            if (Stem(s.ToCharArray(), s.Length))
-            {
-                return ToString();
-            }
-            else
-                return s;
-        }
-        
-        /// <summary>Stem a word contained in a char[].  Returns true if the stemming process
-        /// resulted in a word different from the input.  You can retrieve the
-        /// result with getResultLength()/getResultBuffer() or toString().
-        /// </summary>
-        public virtual bool Stem(char[] word)
-        {
-            return Stem(word, word.Length);
-        }
-        
-        /// <summary>Stem a word contained in a portion of a char[] array.  Returns
-        /// true if the stemming process resulted in a word different from
-        /// the input.  You can retrieve the result with
-        /// getResultLength()/getResultBuffer() or toString().
-        /// </summary>
-        public virtual bool Stem(char[] wordBuffer, int offset, int wordLen)
-        {
-            Reset();
-            if (b.Length < wordLen)
-            {
-                var new_b = new char[wordLen + EXTRA];
-                b = new_b;
-            }
-            Array.Copy(wordBuffer, offset, b, 0, wordLen);
-            i = wordLen;
-            return Stem(0);
-        }
-        
-        /// <summary>Stem a word contained in a leading portion of a char[] array.
-        /// Returns true if the stemming process resulted in a word different
-        /// from the input.  You can retrieve the result with
-        /// getResultLength()/getResultBuffer() or toString().
-        /// </summary>
-        public virtual bool Stem(char[] word, int wordLen)
-        {
-            return Stem(word, 0, wordLen);
-        }
-        
-        /// <summary>Stem the word placed into the Stemmer buffer through calls to add().
-        /// Returns true if the stemming process resulted in a word different
-        /// from the input.  You can retrieve the result with
-        /// getResultLength()/getResultBuffer() or toString().
-        /// </summary>
-        public virtual bool Stem()
-        {
-            return Stem(0);
-        }
-        
-        public virtual bool Stem(int i0)
-        {
-            k = i - 1;
-            k0 = i0;
-            if (k > k0 + 1)
-            {
-                Step1(); Step2(); Step3(); Step4(); Step5(); Step6();
-            }
-            // Also, a word is considered dirty if we lopped off letters
-            // Thanks to Ifigenia Vairelles for pointing this out.
-            if (i != k + 1)
-                dirty = true;
-            i = k + 1;
-            return dirty;
-        }
-        
-        /// <summary>Test program for demonstrating the Stemmer.  It reads a file and
-        /// stems each word, writing the result to standard out.
-        /// Usage: Stemmer file-name
-        /// </summary>
-        [STAThread]
-        public static void  Main(System.String[] args)
-        {
-            var s = new PorterStemmer();
-            
-            for (int i = 0; i < args.Length; i++)
-            {
-                try
-                {
-                    System.IO.Stream in_Renamed = new System.IO.FileStream(args[i], System.IO.FileMode.Open, System.IO.FileAccess.Read);
-                    var buffer = new byte[1024];
-
-                    int bufferLen = in_Renamed.Read(buffer, 0, buffer.Length);
-                    int offset = 0;
-                    s.Reset();
-                    
-                    while (true)
-                    {
-                        int ch;
-                        if (offset < bufferLen)
-                            ch = buffer[offset++];
-                        else
-                        {
-                            bufferLen = in_Renamed.Read(buffer, 0, buffer.Length);
-                            offset = 0;
-                            if (bufferLen < 0)
-                                ch = - 1;
-                            else
-                                ch = buffer[offset++];
-                        }
-                        
-                        if (Char.IsLetter((char) ch))
-                        {
-                            s.Add(Char.ToLower((char) ch));
-                        }
-                        else
-                        {
-                            s.Stem();
-                            Console.Out.Write(s.ToString());
-                            s.Reset();
-                            if (ch < 0)
-                                break;
-                            else
-                            {
-                                System.Console.Out.Write((char) ch);
-                            }
-                        }
-                    }
-                    
-                    in_Renamed.Close();
-                }
-                catch (System.IO.IOException)
-                {
-                    Console.Out.WriteLine("error reading " + args[i]);
-                }
-            }
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8a97bfcf/src/Lucene.Net.Core/Analysis/SimpleAnalyzer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Analysis/SimpleAnalyzer.cs b/src/Lucene.Net.Core/Analysis/SimpleAnalyzer.cs
deleted file mode 100644
index 50bc9c1..0000000
--- a/src/Lucene.Net.Core/Analysis/SimpleAnalyzer.cs
+++ /dev/null
@@ -1,45 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-namespace Lucene.Net.Analysis
-{
-    
-    /// <summary>An <see cref="Analyzer" /> that filters <see cref="LetterTokenizer" /> 
-    /// with <see cref="LowerCaseFilter" /> 
-    /// </summary>
-    
-    public sealed class SimpleAnalyzer : Analyzer
-    {
-        public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
-        {
-            return new LowerCaseTokenizer(reader);
-        }
-        
-        public override TokenStream ReusableTokenStream(System.String fieldName, System.IO.TextReader reader)
-        {
-            var tokenizer = (Tokenizer) PreviousTokenStream;
-            if (tokenizer == null)
-            {
-                tokenizer = new LowerCaseTokenizer(reader);
-                PreviousTokenStream = tokenizer;
-            }
-            else
-                tokenizer.Reset(reader);
-            return tokenizer;
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8a97bfcf/src/Lucene.Net.Core/Analysis/Standard/READ_BEFORE_REGENERATING.txt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Analysis/Standard/READ_BEFORE_REGENERATING.txt b/src/Lucene.Net.Core/Analysis/Standard/READ_BEFORE_REGENERATING.txt
deleted file mode 100644
index 7519d54..0000000
--- a/src/Lucene.Net.Core/Analysis/Standard/READ_BEFORE_REGENERATING.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements.  See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-*/
-
-
-WARNING: if you change StandardTokenizerImpl.jflex and need to regenerate
-      the tokenizer, only use Java 1.4 !!!
-      This grammar currently uses constructs (eg :digit:, :letter:) whose 
-      meaning can vary according to the JRE used to run jflex.  See
-      https://issues.apache.org/jira/browse/LUCENE-1126 for details.
-      For current backwards compatibility it is needed to support
-      only Java 1.4 - this will change in Lucene 3.1.

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8a97bfcf/src/Lucene.Net.Core/Analysis/Standard/StandardAnalyzer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Analysis/Standard/StandardAnalyzer.cs b/src/Lucene.Net.Core/Analysis/Standard/StandardAnalyzer.cs
deleted file mode 100644
index bf704be..0000000
--- a/src/Lucene.Net.Core/Analysis/Standard/StandardAnalyzer.cs
+++ /dev/null
@@ -1,174 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using Lucene.Net.Analysis;
-using Lucene.Net.Util;
-using Version = Lucene.Net.Util.Version;
-
-namespace Lucene.Net.Analysis.Standard
-{
-    
-    /// <summary> Filters <see cref="StandardTokenizer" /> with <see cref="StandardFilter" />,
-    /// <see cref="LowerCaseFilter" /> and <see cref="StopFilter" />, using a list of English stop
-    /// words.
-    /// 
-    /// <a name="version"/>
-    /// <p/>
-    /// You must specify the required <see cref="Version" /> compatibility when creating
-    /// StandardAnalyzer:
-    /// <list type="bullet">
-    /// <item>As of 2.9, StopFilter preserves position increments</item>
-    /// <item>As of 2.4, Tokens incorrectly identified as acronyms are corrected (see
-    /// <a href="https://issues.apache.org/jira/browse/LUCENE-1068">LUCENE-1608</a>)</item>
-    /// </list>
-    /// </summary>
-    public class StandardAnalyzer : Analyzer
-    {
-        private ISet<string> stopSet;
-        
-        /// <summary> Specifies whether deprecated acronyms should be replaced with HOST type.
-        /// See <a href="https://issues.apache.org/jira/browse/LUCENE-1068">https://issues.apache.org/jira/browse/LUCENE-1068</a>
-        /// </summary>
-        private bool replaceInvalidAcronym, enableStopPositionIncrements;
-
-        /// <summary>An unmodifiable set containing some common English words that are usually not
-        /// useful for searching. 
-        /// </summary>
-        public static readonly ISet<string> STOP_WORDS_SET;
-        private Version matchVersion;
-        
-        /// <summary>Builds an analyzer with the default stop words (<see cref="STOP_WORDS_SET" />).
-        /// </summary>
-        /// <param name="matchVersion">Lucene version to match see <see cref="Version">above</see></param>
-        public StandardAnalyzer(Version matchVersion)
-            : this(matchVersion, STOP_WORDS_SET)
-        { }
-        
-        /// <summary>Builds an analyzer with the given stop words.</summary>
-        /// <param name="matchVersion">Lucene version to match See <see cref="Version">above</see> />
-        ///
-        /// </param>
-        /// <param name="stopWords">stop words 
-        /// </param>
-        public StandardAnalyzer(Version matchVersion, ISet<string> stopWords)
-        {
-            stopSet = stopWords;
-            SetOverridesTokenStreamMethod<StandardAnalyzer>();
-            enableStopPositionIncrements = StopFilter.GetEnablePositionIncrementsVersionDefault(matchVersion);
-            replaceInvalidAcronym = matchVersion.OnOrAfter(Version.LUCENE_24);
-            this.matchVersion = matchVersion;
-        }
-        
-        /// <summary>Builds an analyzer with the stop words from the given file.</summary>
-        /// <seealso cref="WordlistLoader.GetWordSet(System.IO.FileInfo)">
-        /// </seealso>
-        /// <param name="matchVersion">Lucene version to match See <see cref="Version">above</see> />
-        ///
-        /// </param>
-        /// <param name="stopwords">File to read stop words from 
-        /// </param>
-        public StandardAnalyzer(Version matchVersion, System.IO.FileInfo stopwords)
-            : this (matchVersion, WordlistLoader.GetWordSet(stopwords))
-        {
-        }
-        
-        /// <summary>Builds an analyzer with the stop words from the given reader.</summary>
-        /// <seealso cref="WordlistLoader.GetWordSet(System.IO.TextReader)">
-        /// </seealso>
-        /// <param name="matchVersion">Lucene version to match See <see cref="Version">above</see> />
-        ///
-        /// </param>
-        /// <param name="stopwords">Reader to read stop words from 
-        /// </param>
-        public StandardAnalyzer(Version matchVersion, System.IO.TextReader stopwords)
-            : this(matchVersion, WordlistLoader.GetWordSet(stopwords))
-        { }
-        
-        /// <summary>Constructs a <see cref="StandardTokenizer" /> filtered by a <see cref="StandardFilter" />
-        ///, a <see cref="LowerCaseFilter" /> and a <see cref="StopFilter" />. 
-        /// </summary>
-        public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
-        {
-            StandardTokenizer tokenStream = new StandardTokenizer(matchVersion, reader);
-            tokenStream.MaxTokenLength = maxTokenLength;
-            TokenStream result = new StandardFilter(tokenStream);
-            result = new LowerCaseFilter(result);
-            result = new StopFilter(enableStopPositionIncrements, result, stopSet);
-            return result;
-        }
-        
-        private sealed class SavedStreams
-        {
-            internal StandardTokenizer tokenStream;
-            internal TokenStream filteredTokenStream;
-        }
-        
-        /// <summary>Default maximum allowed token length </summary>
-        public const int DEFAULT_MAX_TOKEN_LENGTH = 255;
-        
-        private int maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH;
-
-        /// <summary> Set maximum allowed token length.  If a token is seen
-        /// that exceeds this length then it is discarded.  This
-        /// setting only takes effect the next time tokenStream or
-        /// reusableTokenStream is called.
-        /// </summary>
-        public virtual int MaxTokenLength
-        {
-            get { return maxTokenLength; }
-            set { maxTokenLength = value; }
-        }
-
-        public override TokenStream ReusableTokenStream(System.String fieldName, System.IO.TextReader reader)
-        {
-            if (overridesTokenStreamMethod)
-            {
-                // LUCENE-1678: force fallback to tokenStream() if we
-                // have been subclassed and that subclass overrides
-                // tokenStream but not reusableTokenStream
-                return TokenStream(fieldName, reader);
-            }
-            SavedStreams streams = (SavedStreams) PreviousTokenStream;
-            if (streams == null)
-            {
-                streams = new SavedStreams();
-                PreviousTokenStream = streams;
-                streams.tokenStream = new StandardTokenizer(matchVersion, reader);
-                streams.filteredTokenStream = new StandardFilter(streams.tokenStream);
-                streams.filteredTokenStream = new LowerCaseFilter(streams.filteredTokenStream);
-                streams.filteredTokenStream = new StopFilter(enableStopPositionIncrements, 
-                                                             streams.filteredTokenStream, stopSet);
-            }
-            else
-            {
-                streams.tokenStream.Reset(reader);
-            }
-            streams.tokenStream.MaxTokenLength = maxTokenLength;
-            
-            streams.tokenStream.SetReplaceInvalidAcronym(replaceInvalidAcronym);
-            
-            return streams.filteredTokenStream;
-        }
-        static StandardAnalyzer()
-        {
-            STOP_WORDS_SET = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8a97bfcf/src/Lucene.Net.Core/Analysis/Standard/StandardFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Analysis/Standard/StandardFilter.cs b/src/Lucene.Net.Core/Analysis/Standard/StandardFilter.cs
deleted file mode 100644
index f4a1c56..0000000
--- a/src/Lucene.Net.Core/Analysis/Standard/StandardFilter.cs
+++ /dev/null
@@ -1,88 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-using System;
-using Lucene.Net.Analysis.Tokenattributes;
-using Token = Lucene.Net.Analysis.Token;
-using TokenFilter = Lucene.Net.Analysis.TokenFilter;
-using TokenStream = Lucene.Net.Analysis.TokenStream;
-
-namespace Lucene.Net.Analysis.Standard
-{
-    
-    /// <summary>Normalizes tokens extracted with <see cref="StandardTokenizer" />. </summary>
-    
-    public sealed class StandardFilter:TokenFilter
-    {
-        
-        
-        /// <summary>Construct filtering <i>in</i>. </summary>
-        public StandardFilter(TokenStream in_Renamed):base(in_Renamed)
-        {
-            termAtt = AddAttribute<ITermAttribute>();
-            typeAtt = AddAttribute<ITypeAttribute>();
-        }
-        
-        private static readonly System.String APOSTROPHE_TYPE;
-        private static readonly System.String ACRONYM_TYPE;
-        
-        // this filters uses attribute type
-        private ITypeAttribute typeAtt;
-        private ITermAttribute termAtt;
-        
-        /// <summary>Returns the next token in the stream, or null at EOS.
-        /// <p/>Removes <tt>'s</tt> from the end of words.
-        /// <p/>Removes dots from acronyms.
-        /// </summary>
-        public override bool IncrementToken()
-        {
-            if (!input.IncrementToken())
-            {
-                return false;
-            }
-            
-            char[] buffer = termAtt.TermBuffer();
-            int bufferLength = termAtt.TermLength();
-            System.String type = typeAtt.Type;
-            
-            if ((System.Object) type == (System.Object) APOSTROPHE_TYPE && bufferLength >= 2 && buffer[bufferLength - 2] == '\'' && (buffer[bufferLength - 1] == 's' || buffer[bufferLength - 1] == 'S'))
-            {
-                // Strip last 2 characters off
-                termAtt.SetTermLength(bufferLength - 2);
-            }
-            else if ((System.Object) type == (System.Object) ACRONYM_TYPE)
-            {
-                // remove dots
-                int upto = 0;
-                for (int i = 0; i < bufferLength; i++)
-                {
-                    char c = buffer[i];
-                    if (c != '.')
-                        buffer[upto++] = c;
-                }
-                termAtt.SetTermLength(upto);
-            }
-            
-            return true;
-        }
-        static StandardFilter()
-        {
-            APOSTROPHE_TYPE = StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.APOSTROPHE];
-            ACRONYM_TYPE = StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.ACRONYM];
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8a97bfcf/src/Lucene.Net.Core/Analysis/Standard/StandardTokenizer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Analysis/Standard/StandardTokenizer.cs b/src/Lucene.Net.Core/Analysis/Standard/StandardTokenizer.cs
deleted file mode 100644
index 8f25c7c..0000000
--- a/src/Lucene.Net.Core/Analysis/Standard/StandardTokenizer.cs
+++ /dev/null
@@ -1,232 +0,0 @@
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-using System;
-using Lucene.Net.Analysis.Tokenattributes;
-using Lucene.Net.Util;
-using CharReader = Lucene.Net.Analysis.CharReader;
-using Token = Lucene.Net.Analysis.Token;
-using Tokenizer = Lucene.Net.Analysis.Tokenizer;
-using AttributeSource = Lucene.Net.Util.AttributeSource;
-using Version = Lucene.Net.Util.Version;
-
-namespace Lucene.Net.Analysis.Standard
-{
-    
-    /// <summary>A grammar-based tokenizer constructed with JFlex
-    /// 
-    /// <p/> This should be a good tokenizer for most European-language documents:
-    /// 
-    /// <list type="bullet">
-    /// <item>Splits words at punctuation characters, removing punctuation. However, a 
-    /// dot that's not followed by whitespace is considered part of a token.</item>
-    /// <item>Splits words at hyphens, unless there's a number in the token, in which case
-    /// the whole token is interpreted as a product number and is not split.</item>
-    /// <item>Recognizes email addresses and internet hostnames as one token.</item>
-    /// </list>
-    /// 
-    /// <p/>Many applications have specific tokenizer needs.  If this tokenizer does
-    /// not suit your application, please consider copying this source code
-    /// directory to your project and maintaining your own grammar-based tokenizer.
-    /// 
-    /// <a name="version"/>
-    /// <p/>
-    /// You must specify the required <see cref="Version" /> compatibility when creating
-    /// StandardAnalyzer:
-    /// <list type="bullet">
-    /// <item>As of 2.4, Tokens incorrectly identified as acronyms are corrected (see
-    /// <a href="https://issues.apache.org/jira/browse/LUCENE-1068">LUCENE-1608</a></item>
-    /// </list>
-    /// </summary>
-    
-    public sealed class StandardTokenizer:Tokenizer
-    {
-        private void  InitBlock()
-        {
-            maxTokenLength = StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH;
-        }
-        /// <summary>A private instance of the JFlex-constructed scanner </summary>
-        private StandardTokenizerImpl scanner;
-        
-        public const int ALPHANUM   = 0;
-        public const int APOSTROPHE = 1;
-        public const int ACRONYM    = 2;
-        public const int COMPANY    = 3;
-        public const int EMAIL      = 4;
-        public const int HOST       = 5;
-        public const int NUM        = 6;
-        public const int CJ         = 7;
-        
-        /// <deprecated> this solves a bug where HOSTs that end with '.' are identified
-        /// as ACRONYMs.
-        /// </deprecated>
-        [Obsolete("this solves a bug where HOSTs that end with '.' are identified as ACRONYMs.")]
-        public const int ACRONYM_DEP = 8;
-        
-        /// <summary>String token types that correspond to token type int constants </summary>
-        public static readonly System.String[] TOKEN_TYPES = new System.String[]{"<ALPHANUM>", "<APOSTROPHE>", "<ACRONYM>", "<COMPANY>", "<EMAIL>", "<HOST>", "<NUM>", "<CJ>", "<ACRONYM_DEP>"};
-        
-        private bool replaceInvalidAcronym;
-        
-        private int maxTokenLength;
-
-        /// <summary>Set the max allowed token length.  Any token longer
-        /// than this is skipped. 
-        /// </summary>
-        public int MaxTokenLength
-        {
-            get { return maxTokenLength; }
-            set { this.maxTokenLength = value; }
-        }
-
-        /// <summary> Creates a new instance of the
-        /// <see cref="Lucene.Net.Analysis.Standard.StandardTokenizer" />. Attaches
-        /// the <c>input</c> to the newly created JFlex scanner.
-        /// 
-        /// </summary>
-        /// <param name="matchVersion"></param>
-        /// <param name="input">The input reader
-        /// 
-        /// See http://issues.apache.org/jira/browse/LUCENE-1068
-        /// </param>
-        public StandardTokenizer(Version matchVersion, System.IO.TextReader input):base()
-        {
-            InitBlock();
-            this.scanner = new StandardTokenizerImpl(input);
-            Init(input, matchVersion);
-        }
-
-        /// <summary> Creates a new StandardTokenizer with a given <see cref="AttributeSource" />.</summary>
-        public StandardTokenizer(Version matchVersion, AttributeSource source, System.IO.TextReader input):base(source)
-        {
-            InitBlock();
-            this.scanner = new StandardTokenizerImpl(input);
-            Init(input, matchVersion);
-        }
-        
-        /// <summary> Creates a new StandardTokenizer with a given
-        /// <see cref="Lucene.Net.Util.AttributeSource.AttributeFactory" />
-        /// </summary>
-        public StandardTokenizer(Version matchVersion, AttributeFactory factory, System.IO.TextReader input):base(factory)
-        {
-            InitBlock();
-            this.scanner = new StandardTokenizerImpl(input);
-            Init(input, matchVersion);
-        }
-        
-        private void  Init(System.IO.TextReader input, Version matchVersion)
-        {
-            if (matchVersion.OnOrAfter(Version.LUCENE_24))
-            {
-                replaceInvalidAcronym = true;
-            }
-            else
-            {
-                replaceInvalidAcronym = false;
-            }
-            this.input = input;
-            termAtt = AddAttribute<ITermAttribute>();
-            offsetAtt = AddAttribute<IOffsetAttribute>();
-            posIncrAtt = AddAttribute<IPositionIncrementAttribute>();
-            typeAtt = AddAttribute<ITypeAttribute>();
-        }
-        
-        // this tokenizer generates three attributes:
-        // offset, positionIncrement and type
-        private ITermAttribute termAtt;
-        private IOffsetAttribute offsetAtt;
-        private IPositionIncrementAttribute posIncrAtt;
-        private ITypeAttribute typeAtt;
-        
-        ///<summary>
-        /// (non-Javadoc)
-        /// <see cref="Lucene.Net.Analysis.TokenStream.IncrementToken()" />
-        ///</summary>
-        public override bool IncrementToken()
-        {
-            ClearAttributes();
-            int posIncr = 1;
-            
-            while (true)
-            {
-                int tokenType = scanner.GetNextToken();
-                
-                if (tokenType == StandardTokenizerImpl.YYEOF)
-                {
-                    return false;
-                }
-                
-                if (scanner.Yylength() <= maxTokenLength)
-                {
-                    posIncrAtt.PositionIncrement = posIncr;
-                    scanner.GetText(termAtt);
-                    int start = scanner.Yychar();
-                    offsetAtt.SetOffset(CorrectOffset(start), CorrectOffset(start + termAtt.TermLength()));
-                    // This 'if' should be removed in the next release. For now, it converts
-                    // invalid acronyms to HOST. When removed, only the 'else' part should
-                    // remain.
-                    if (tokenType == StandardTokenizerImpl.ACRONYM_DEP)
-                    {
-                        if (replaceInvalidAcronym)
-                        {
-                            typeAtt.Type = StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.HOST];
-                            termAtt.SetTermLength(termAtt.TermLength() - 1); // remove extra '.'
-                        }
-                        else
-                        {
-                            typeAtt.Type = StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.ACRONYM];
-                        }
-                    }
-                    else
-                    {
-                        typeAtt.Type = StandardTokenizerImpl.TOKEN_TYPES[tokenType];
-                    }
-                    return true;
-                }
-                // When we skip a too-long term, we still increment the
-                // position increment
-                else
-                    posIncr++;
-            }
-        }
-        
-        public override void  End()
-        {
-            // set final offset
-            int finalOffset = CorrectOffset(scanner.Yychar() + scanner.Yylength());
-            offsetAtt.SetOffset(finalOffset, finalOffset);
-        }
-        
-        public override void  Reset(System.IO.TextReader reader)
-        {
-            base.Reset(reader);
-            scanner.Reset(reader);
-        }
-        
-        /// <summary>
-        /// Remove in 3.X and make true the only valid value
-        /// See https://issues.apache.org/jira/browse/LUCENE-1068
-        /// </summary>
-        /// <param name="replaceInvalidAcronym">Set to true to replace mischaracterized acronyms as HOST.
-        /// </param>
-        [Obsolete("Remove in 3.X and make true the only valid value. See https://issues.apache.org/jira/browse/LUCENE-1068")]
-        public void  SetReplaceInvalidAcronym(bool replaceInvalidAcronym)
-        {
-            this.replaceInvalidAcronym = replaceInvalidAcronym;
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/8a97bfcf/src/Lucene.Net.Core/Analysis/Standard/StandardTokenizerImpl.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Analysis/Standard/StandardTokenizerImpl.cs b/src/Lucene.Net.Core/Analysis/Standard/StandardTokenizerImpl.cs
deleted file mode 100644
index cf2a81e..0000000
--- a/src/Lucene.Net.Core/Analysis/Standard/StandardTokenizerImpl.cs
+++ /dev/null
@@ -1,707 +0,0 @@
-/* The following code was generated by JFlex 1.4.1 on 9/4/08 6:49 PM */
-/* 
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-/*
-    NOTE: if you change StandardTokenizerImpl.jflex and need to regenerate the tokenizer,
-    the tokenizer, only use Java 1.4 !!!
-    This grammar currently uses constructs (eg :digit:, :letter:) whose
-    meaning can vary according to the JRE used to run jflex.  See
-    https://issues.apache.org/jira/browse/LUCENE-1126 for details.
-    For current backwards compatibility it is needed to support
-    only Java 1.4 - this will change in Lucene 3.1.
-*/
-
-using System;
-using Lucene.Net.Analysis.Tokenattributes;
-using Token = Lucene.Net.Analysis.Token;
-
-namespace Lucene.Net.Analysis.Standard
-{
-    
-    
-    /// <summary> This class is a scanner generated by 
-    /// <a href="http://www.jflex.de/">JFlex</a> 1.4.1
-    /// on 9/4/08 6:49 PM from the specification file
-    /// <tt>/tango/mike/src/lucene.standarddigit/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex</tt>
-    /// </summary>
-    class StandardTokenizerImpl
-    {
-        
-        /// <summary>This character denotes the end of file </summary>
-        public const int YYEOF = - 1;
-        
-        /// <summary>initial size of the lookahead buffer </summary>
-        private const int ZZ_BUFFERSIZE = 16384;
-        
-        /// <summary>lexical states </summary>
-        public const int YYINITIAL = 0;
-        
-        /// <summary> Translates characters to character classes</summary>
-        private const System.String ZZ_CMAP_PACKED = "\x0009\x0000\x0001\x0000\x0001\x000D\x0001\x0000\x0001\x0000\x0001\x000C\x0012\x0000\x0001\x0000\x0005\x0000\x0001\x0005" + "\x0001\x0003\x0004\x0000\x0001\x0009\x0001\x0007\x0001\x0004\x0001\x0009\x000A\x0002\x0006\x0000\x0001\x0006\x001A\x000A" + "\x0004\x0000\x0001\x0008\x0001\x0000\x001A\x000A\x002F\x0000\x0001\x000A\x000A\x0000\x0001\x000A\x0004\x0000\x0001\x000A" + "\x0005\x0000\x0017\x000A\x0001\x0000\x001F\x000A\x0001\x0000\u0128\x000A\x0002\x0000\x0012\x000A\x001C\x0000\x005E\x000A" + "\x0002\x0000\x0009\x000A\x0002\x0000\x0007\x000A\x000E\x0000\x0002\x000A\x000E\x0000\x0005\x000A\x0009\x0000\x0001\x000A" + "\x008B\x0000\x0001\x000A\x000B\x0000\x0001\x000A\x0001\x0000\x0003\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0014\x000A" + "\x0001\x0000\x002C\x000A\x0001\x0000\x0008\x000A\x0002\x0000\x001A\x000A\x000C\x0000\x0082\x000A\x000A\x0000\x0039\x000A" + "\x0002\x0000\x0002\x000A\x0002\x0000\x0002\x000A\x0003\x0000\x0026\x
 000A\x0002\x0000\x0002\x000A\x0037\x0000\x0026\x000A" + "\x0002\x0000\x0001\x000A\x0007\x0000\x0027\x000A\x0048\x0000\x001B\x000A\x0005\x0000\x0003\x000A\x002E\x0000\x001A\x000A" + "\x0005\x0000\x000B\x000A\x0015\x0000\x000A\x0002\x0007\x0000\x0063\x000A\x0001\x0000\x0001\x000A\x000F\x0000\x0002\x000A" + "\x0009\x0000\x000A\x0002\x0003\x000A\x0013\x0000\x0001\x000A\x0001\x0000\x001B\x000A\x0053\x0000\x0026\x000A\u015f\x0000" + "\x0035\x000A\x0003\x0000\x0001\x000A\x0012\x0000\x0001\x000A\x0007\x0000\x000A\x000A\x0004\x0000\x000A\x0002\x0015\x0000" + "\x0008\x000A\x0002\x0000\x0002\x000A\x0002\x0000\x0016\x000A\x0001\x0000\x0007\x000A\x0001\x0000\x0001\x000A\x0003\x0000" + "\x0004\x000A\x0022\x0000\x0002\x000A\x0001\x0000\x0003\x000A\x0004\x0000\x000A\x0002\x0002\x000A\x0013\x0000\x0006\x000A" + "\x0004\x0000\x0002\x000A\x0002\x0000\x0016\x000A\x0001\x0000\x0007\x000A\x0001\x0000\x0002\x000A\x0001\x0000\x0002\x000A" + 
-            "\x0001\x0000\x0002\x000A\x001F\x0000\x0004\x000A\x0001\x0000\x0001\x000A\x0007\x0000\x000A\x0002\x0002\x0000\x0003\x000A" + "\x0010\x0000\x0007\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0003\x000A\x0001\x0000\x0016\x000A\x0001\x0000\x0007\x000A" + "\x0001\x0000\x0002\x000A\x0001\x0000\x0005\x000A\x0003\x0000\x0001\x000A\x0012\x0000\x0001\x000A\x000F\x0000\x0001\x000A" + "\x0005\x0000\x000A\x0002\x0015\x0000\x0008\x000A\x0002\x0000\x0002\x000A\x0002\x0000\x0016\x000A\x0001\x0000\x0007\x000A" + "\x0001\x0000\x0002\x000A\x0002\x0000\x0004\x000A\x0003\x0000\x0001\x000A\x001E\x0000\x0002\x000A\x0001\x0000\x0003\x000A" + "\x0004\x0000\x000A\x0002\x0015\x0000\x0006\x000A\x0003\x0000\x0003\x000A\x0001\x0000\x0004\x000A\x0003\x0000\x0002\x000A" + "\x0001\x0000\x0001\x000A\x0001\x0000\x0002\x000A\x0003\x0000\x0002\x000A\x0003\x0000\x0003\x000A\x0003\x0000\x0008\x000A" + "\x0001\x0000\x0003\x000A\x002D\x0000\x0009\x0002\x0015\x0000\x0008\x000A\x0001\x0000\x0003\x000A\x0001\x0000\
 x0017\x000A" + "\x0001\x0000\x000A\x000A\x0001\x0000\x0005\x000A\x0026\x0000\x0002\x000A\x0004\x0000\x000A\x0002\x0015\x0000\x0008\x000A" + "\x0001\x0000\x0003\x000A\x0001\x0000\x0017\x000A\x0001\x0000\x000A\x000A\x0001\x0000\x0005\x000A\x0024\x0000\x0001\x000A" + "\x0001\x0000\x0002\x000A\x0004\x0000\x000A\x0002\x0015\x0000\x0008\x000A\x0001\x0000\x0003\x000A\x0001\x0000\x0017\x000A" + "\x0001\x0000\x0010\x000A\x0026\x0000\x0002\x000A\x0004\x0000\x000A\x0002\x0015\x0000\x0012\x000A\x0003\x0000\x0018\x000A" + "\x0001\x0000\x0009\x000A\x0001\x0000\x0001\x000A\x0002\x0000\x0007\x000A\x0039\x0000\x0001\x0001\x0030\x000A\x0001\x0001" + "\x0002\x000A\x000C\x0001\x0007\x000A\x0009\x0001\x000A\x0002\x0027\x0000\x0002\x000A\x0001\x0000\x0001\x000A\x0002\x0000" + "\x0002\x000A\x0001\x0000\x0001\x000A\x0002\x0000\x0001\x000A\x0006\x0000\x0004\x000A\x0001\x0000\x0007\x000A\x0001\x0000" + "\x0003\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0001\x000A\x0002\x0000\x0002\x000A\x0001\x0000\x0004\x00
 0A\x0001\x0000" + 
-            "\x0002\x000A\x0009\x0000\x0001\x000A\x0002\x0000\x0005\x000A\x0001\x0000\x0001\x000A\x0009\x0000\x000A\x0002\x0002\x0000" + "\x0002\x000A\x0022\x0000\x0001\x000A\x001F\x0000\x000A\x0002\x0016\x0000\x0008\x000A\x0001\x0000\x0022\x000A\x001D\x0000" + "\x0004\x000A\x0074\x0000\x0022\x000A\x0001\x0000\x0005\x000A\x0001\x0000\x0002\x000A\x0015\x0000\x000A\x0002\x0006\x0000" + "\x0006\x000A\x004A\x0000\x0026\x000A\x000A\x0000\x0027\x000A\x0009\x0000\x005A\x000A\x0005\x0000\x0044\x000A\x0005\x0000" + "\x0052\x000A\x0006\x0000\x0007\x000A\x0001\x0000\x003F\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0004\x000A\x0002\x0000" + "\x0007\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0004\x000A\x0002\x0000\x0027\x000A\x0001\x0000\x0001\x000A\x0001\x0000" + "\x0004\x000A\x0002\x0000\x001F\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0004\x000A\x0002\x0000\x0007\x000A\x0001\x0000" + "\x0001\x000A\x0001\x0000\x0004\x000A\x0002\x0000\x0007\x000A\x0001\x0000\x0007\x000A\x0001\x0000\x0017\x000A\
 x0001\x0000" + "\x001F\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0004\x000A\x0002\x0000\x0007\x000A\x0001\x0000\x0027\x000A\x0001\x0000" + "\x0013\x000A\x000E\x0000\x0009\x0002\x002E\x0000\x0055\x000A\x000C\x0000\u026c\x000A\x0002\x0000\x0008\x000A\x000A\x0000" + "\x001A\x000A\x0005\x0000\x004B\x000A\x0095\x0000\x0034\x000A\x002C\x0000\x000A\x0002\x0026\x0000\x000A\x0002\x0006\x0000" + "\x0058\x000A\x0008\x0000\x0029\x000A\u0557\x0000\x009C\x000A\x0004\x0000\x005A\x000A\x0006\x0000\x0016\x000A\x0002\x0000" + "\x0006\x000A\x0002\x0000\x0026\x000A\x0002\x0000\x0006\x000A\x0002\x0000\x0008\x000A\x0001\x0000\x0001\x000A\x0001\x0000" + "\x0001\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x001F\x000A\x0002\x0000\x0035\x000A\x0001\x0000\x0007\x000A\x0001\x0000" + "\x0001\x000A\x0003\x0000\x0003\x000A\x0001\x0000\x0007\x000A\x0003\x0000\x0004\x000A\x0002\x0000\x0006\x000A\x0004\x0000" + "\x000D\x000A\x0005\x0000\x0003\x000A\x0001\x0000\x0007\x000A\x0082\x0000\x0001\x000A\x0082\x0000\x0001\x00
 0A\x0004\x0000" + 
-            "\x0001\x000A\x0002\x0000\x000A\x000A\x0001\x0000\x0001\x000A\x0003\x0000\x0005\x000A\x0006\x0000\x0001\x000A\x0001\x0000" + "\x0001\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0004\x000A\x0001\x0000\x0003\x000A\x0001\x0000\x0007\x000A\u0ecb\x0000" + "\x0002\x000A\x002A\x0000\x0005\x000A\x000A\x0000\x0001\x000B\x0054\x000B\x0008\x000B\x0002\x000B\x0002\x000B\x005A\x000B" + "\x0001\x000B\x0003\x000B\x0006\x000B\x0028\x000B\x0003\x000B\x0001\x0000\x005E\x000A\x0011\x0000\x0018\x000A\x0038\x0000" + "\x0010\x000B\u0100\x0000\x0080\x000B\x0080\x0000\u19b6\x000B\x000A\x000B\x0040\x0000\u51a6\x000B\x005A\x000B\u048d\x000A" + "\u0773\x0000\u2ba4\x000A\u215c\x0000\u012e\x000B\x00D2\x000B\x0007\x000A\x000C\x0000\x0005\x000A\x0005\x0000\x0001\x000A" + "\x0001\x0000\x000A\x000A\x0001\x0000\x000D\x000A\x0001\x0000\x0005\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0002\x000A" + "\x0001\x0000\x0002\x000A\x0001\x0000\x006C\x000A\x0021\x0000\u016b\x000A\x0012\x0000\x0040\x000A\x0002\x0000\
 x0036\x000A" + "\x0028\x0000\x000C\x000A\x0074\x0000\x0003\x000A\x0001\x0000\x0001\x000A\x0001\x0000\x0087\x000A\x0013\x0000\x000A\x0002" + "\x0007\x0000\x001A\x000A\x0006\x0000\x001A\x000A\x000A\x0000\x0001\x000B\x003A\x000B\x001F\x000A\x0003\x0000\x0006\x000A" + "\x0002\x0000\x0006\x000A\x0002\x0000\x0006\x000A\x0002\x0000\x0003\x000A\x0023\x0000";
-        
-        /// <summary> Translates characters to character classes</summary>
-        private static readonly char[] ZZ_CMAP = ZzUnpackCMap(ZZ_CMAP_PACKED);
-        
-        /// <summary> Translates DFA states to action switch labels.</summary>
-        private static readonly int[] ZZ_ACTION = ZzUnpackAction();
-        
-        private const System.String ZZ_ACTION_PACKED_0 = "\x0001\x0000\x0001\x0001\x0003\x0002\x0001\x0003\x0001\x0001\x000B\x0000\x0001\x0002\x0003\x0004" + "\x0002\x0000\x0001\x0005\x0001\x0000\x0001\x0005\x0003\x0004\x0006\x0005\x0001\x0006\x0001\x0004" + "\x0002\x0007\x0001\x0008\x0001\x0000\x0001\x0008\x0003\x0000\x0002\x0008\x0001\x0009\x0001\x000A" + "\x0001\x0004";
-        
-        private static int[] ZzUnpackAction()
-        {
-            int[] result = new int[51];
-            int offset = 0;
-            offset = ZzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
-            return result;
-        }
-        
-        private static int ZzUnpackAction(System.String packed, int offset, int[] result)
-        {
-            int i = 0; /* index in packed string  */
-            int j = offset; /* index in unpacked array */
-            int l = packed.Length;
-            while (i < l)
-            {
-                int count = packed[i++];
-                int value_Renamed = packed[i++];
-                do 
-                    result[j++] = value_Renamed;
-                while (--count > 0);
-            }
-            return j;
-        }
-        
-        
-        /// <summary> Translates a state to a row index in the transition table</summary>
-        private static readonly int[] ZZ_ROWMAP = ZzUnpackRowMap();
-        
-        private const System.String ZZ_ROWMAP_PACKED_0 = "\x0000\x0000\x0000\x000E\x0000\x001C\x0000\x002A\x0000\x0038\x0000\x000E\x0000\x0046\x0000\x0054" + "\x0000\x0062\x0000\x0070\x0000\x007E\x0000\x008C\x0000\x009A\x0000\x00A8\x0000\x00B6\x0000\x00C4" + "\x0000\x00D2\x0000\x00E0\x0000\x00EE\x0000\x00FC\x0000\u010a\x0000\u0118\x0000\u0126\x0000\u0134" + "\x0000\u0142\x0000\u0150\x0000\u015e\x0000\u016c\x0000\u017a\x0000\u0188\x0000\u0196\x0000\u01a4" + "\x0000\u01b2\x0000\u01c0\x0000\u01ce\x0000\u01dc\x0000\u01ea\x0000\u01f8\x0000\x00D2\x0000\u0206" + "\x0000\u0214\x0000\u0222\x0000\u0230\x0000\u023e\x0000\u024c\x0000\u025a\x0000\x0054\x0000\x008C" + "\x0000\u0268\x0000\u0276\x0000\u0284";
-        
-        private static int[] ZzUnpackRowMap()
-        {
-            int[] result = new int[51];
-            int offset = 0;
-            offset = ZzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
-            return result;
-        }
-        
-        private static int ZzUnpackRowMap(System.String packed, int offset, int[] result)
-        {
-            int i = 0; /* index in packed string  */
-            int j = offset; /* index in unpacked array */
-            int l = packed.Length;
-            while (i < l)
-            {
-                int high = packed[i++] << 16;
-                result[j++] = high | packed[i++];
-            }
-            return j;
-        }
-        
-        /// <summary> The transition table of the DFA</summary>
-        private static readonly int[] ZZ_TRANS = ZzUnpackTrans();
-        
-        private const System.String ZZ_TRANS_PACKED_0 = "\x0001\x0002\x0001\x0003\x0001\x0004\x0007\x0002\x0001\x0005\x0001\x0006\x0001\x0007\x0001\x0002" + "\x000F\x0000\x0002\x0003\x0001\x0000\x0001\x0008\x0001\x0000\x0001\x0009\x0002\x000A\x0001\x000B" + "\x0001\x0003\x0004\x0000\x0001\x0003\x0001\x0004\x0001\x0000\x0001\x000C\x0001\x0000\x0001\x0009" + "\x0002\x000D\x0001\x000E\x0001\x0004\x0004\x0000\x0001\x0003\x0001\x0004\x0001\x000F\x0001\x0010" + "\x0001\x0011\x0001\x0012\x0002\x000A\x0001\x000B\x0001\x0013\x0010\x0000\x0001\x0002\x0001\x0000" + "\x0001\x0014\x0001\x0015\x0007\x0000\x0001\x0016\x0004\x0000\x0002\x0017\x0007\x0000\x0001\x0017" + "\x0004\x0000\x0001\x0018\x0001\x0019\x0007\x0000\x0001\x001A\x0005\x0000\x0001\x001B\x0007\x0000" + "\x0001\x000B\x0004\x0000\x0001\x001C\x0001\x001D\x0007\x0000\x0001\x001E\x0004\x0000\x0001\x001F" + "\x0001\x0020\x0007\x0000\x0001\x0021\x0004\x0000\x0001\x0022\x0001\x0023\x0007\x0000\x0001\x0024" + "\x000D\x0000\x0001\x0025\x0004\
 x0000\x0001\x0014\x0001\x0015\x0007\x0000\x0001\x0026\x000D\x0000" + "\x0001\x0027\x0004\x0000\x0002\x0017\x0007\x0000\x0001\x0028\x0004\x0000\x0001\x0003\x0001\x0004" + "\x0001\x000F\x0001\x0008\x0001\x0011\x0001\x0012\x0002\x000A\x0001\x000B\x0001\x0013\x0004\x0000" + "\x0002\x0014\x0001\x0000\x0001\x0029\x0001\x0000\x0001\x0009\x0002\x002A\x0001\x0000\x0001\x0014" + "\x0004\x0000\x0001\x0014\x0001\x0015\x0001\x0000\x0001\x002B\x0001\x0000\x0001\x0009\x0002\x002C" + "\x0001\x002D\x0001\x0015\x0004\x0000\x0001\x0014\x0001\x0015\x0001\x0000\x0001\x0029\x0001\x0000" + "\x0001\x0009\x0002\x002A\x0001\x0000\x0001\x0016\x0004\x0000\x0002\x0017\x0001\x0000\x0001\x002E" + "\x0002\x0000\x0001\x002E\x0002\x0000\x0001\x0017\x0004\x0000\x0002\x0018\x0001\x0000\x0001\x002A" + "\x0001\x0000\x0001\x0009\x0002\x002A\x0001\x0000\x0001\x0018\x0004\x0000\x0001\x0018\x0001\x0019" + "\x0001\x0000\x0001\x002C\x0001\x0000\x0001\x0009\x0002\x002C\x0001\x002D\x0001\x0019\x0004\x0000" + 
-            "\x0001\x0018\x0001\x0019\x0001\x0000\x0001\x002A\x0001\x0000\x0001\x0009\x0002\x002A\x0001\x0000" + "\x0001\x001A\x0005\x0000\x0001\x001B\x0001\x0000\x0001\x002D\x0002\x0000\x0003\x002D\x0001\x001B" + "\x0004\x0000\x0002\x001C\x0001\x0000\x0001\x002F\x0001\x0000\x0001\x0009\x0002\x000A\x0001\x000B" + "\x0001\x001C\x0004\x0000\x0001\x001C\x0001\x001D\x0001\x0000\x0001\x0030\x0001\x0000\x0001\x0009" + "\x0002\x000D\x0001\x000E\x0001\x001D\x0004\x0000\x0001\x001C\x0001\x001D\x0001\x0000\x0001\x002F" + "\x0001\x0000\x0001\x0009\x0002\x000A\x0001\x000B\x0001\x001E\x0004\x0000\x0002\x001F\x0001\x0000" + "\x0001\x000A\x0001\x0000\x0001\x0009\x0002\x000A\x0001\x000B\x0001\x001F\x0004\x0000\x0001\x001F" + "\x0001\x0020\x0001\x0000\x0001\x000D\x0001\x0000\x0001\x0009\x0002\x000D\x0001\x000E\x0001\x0020" + "\x0004\x0000\x0001\x001F\x0001\x0020\x0001\x0000\x0001\x000A\x0001\x0000\x0001\x0009\x0002\x000A" + "\x0001\x000B\x0001\x0021\x0004\x0000\x0002\x0022\x0001\x0000\x0001\x000B\x0
 002\x0000\x0003\x000B" + "\x0001\x0022\x0004\x0000\x0001\x0022\x0001\x0023\x0001\x0000\x0001\x000E\x0002\x0000\x0003\x000E" + "\x0001\x0023\x0004\x0000\x0001\x0022\x0001\x0023\x0001\x0000\x0001\x000B\x0002\x0000\x0003\x000B" + "\x0001\x0024\x0006\x0000\x0001\x000F\x0006\x0000\x0001\x0025\x0004\x0000\x0001\x0014\x0001\x0015" + "\x0001\x0000\x0001\x0031\x0001\x0000\x0001\x0009\x0002\x002A\x0001\x0000\x0001\x0016\x0004\x0000" + "\x0002\x0017\x0001\x0000\x0001\x002E\x0002\x0000\x0001\x002E\x0002\x0000\x0001\x0028\x0004\x0000" + "\x0002\x0014\x0007\x0000\x0001\x0014\x0004\x0000\x0002\x0018\x0007\x0000\x0001\x0018\x0004\x0000" + "\x0002\x001C\x0007\x0000\x0001\x001C\x0004\x0000\x0002\x001F\x0007\x0000\x0001\x001F\x0004\x0000" + "\x0002\x0022\x0007\x0000\x0001\x0022\x0004\x0000\x0002\x0032\x0007\x0000\x0001\x0032\x0004\x0000" + "\x0002\x0014\x0007\x0000\x0001\x0033\x0004\x0000\x0002\x0032\x0001\x0000\x0001\x002E\x0002\x0000" + "\x0001\x002E\x0002\x0000\x0001\x0032\x0004\x0000\x0002\x0014\x
 0001\x0000\x0001\x0031\x0001\x0000" + 
-            "\x0001\x0009\x0002\x002A\x0001\x0000\x0001\x0014\x0003\x0000";
-        
-        private static int[] ZzUnpackTrans()
-        {
-            int[] result = new int[658];
-            int offset = 0;
-            offset = ZzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
-            return result;
-        }
-        
-        private static int ZzUnpackTrans(System.String packed, int offset, int[] result)
-        {
-            int i = 0; /* index in packed string  */
-            int j = offset; /* index in unpacked array */
-            int l = packed.Length;
-            while (i < l)
-            {
-                int count = packed[i++];
-                int value_Renamed = packed[i++];
-                value_Renamed--;
-                do 
-                    result[j++] = value_Renamed;
-                while (--count > 0);
-            }
-            return j;
-        }
-        
-        
-        /* error codes */
-        private const int ZZ_UNKNOWN_ERROR = 0;
-        private const int ZZ_NO_MATCH = 1;
-        private const int ZZ_PUSHBACK_2BIG = 2;
-        
-        /* error messages for the codes above */
-        private static readonly System.String[] ZZ_ERROR_MSG = new System.String[]{"Unkown internal scanner error", "Error: could not match input", "Error: pushback value was too large"};
-        
-        /// <summary> ZZ_ATTRIBUTE[aState] contains the attributes of state <c>aState</c></summary>
-        private static readonly int[] ZZ_ATTRIBUTE = ZzUnpackAttribute();
-        
-        private const System.String ZZ_ATTRIBUTE_PACKED_0 = "\x0001\x0000\x0001\x0009\x0003\x0001\x0001\x0009\x0001\x0001\x000B\x0000\x0004\x0001\x0002\x0000" + "\x0001\x0001\x0001\x0000\x000F\x0001\x0001\x0000\x0001\x0001\x0003\x0000\x0005\x0001";
-        
-        private static int[] ZzUnpackAttribute()
-        {
-            int[] result = new int[51];
-            int offset = 0;
-            offset = ZzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
-            return result;
-        }
-        
-        private static int ZzUnpackAttribute(System.String packed, int offset, int[] result)
-        {
-            int i = 0; /* index in packed string  */
-            int j = offset; /* index in unpacked array */
-            int l = packed.Length;
-            while (i < l)
-            {
-                int count = packed[i++];
-                int value_Renamed = packed[i++];
-                do 
-                    result[j++] = value_Renamed;
-                while (--count > 0);
-            }
-            return j;
-        }
-        
-        /// <summary>the input device </summary>
-        private System.IO.TextReader zzReader;
-        
-        /// <summary>the current state of the DFA </summary>
-        private int zzState;
-        
-        /// <summary>the current lexical state </summary>
-        private int zzLexicalState = YYINITIAL;
-        
-        /// <summary>this buffer contains the current text to be matched and is
-        /// the source of the yytext() string 
-        /// </summary>
-        private char[] zzBuffer = new char[ZZ_BUFFERSIZE];
-        
-        /// <summary>the textposition at the last accepting state </summary>
-        private int zzMarkedPos;
-        
-        /// <summary>the textposition at the last state to be included in yytext </summary>
-        private int zzPushbackPos;
-        
-        /// <summary>the current text position in the buffer </summary>
-        private int zzCurrentPos;
-        
-        /// <summary>startRead marks the beginning of the yytext() string in the buffer </summary>
-        private int zzStartRead;
-        
-        /// <summary>endRead marks the last character in the buffer, that has been read
-        /// from input 
-        /// </summary>
-        private int zzEndRead;
-        
-        /// <summary>number of newlines encountered up to the start of the matched text </summary>
-        private int yyline;
-        
-        /// <summary>the number of characters up to the start of the matched text </summary>
-        private int yychar;
-        
-        /// <summary> the number of characters from the last newline up to the start of the 
-        /// matched text
-        /// </summary>
-        private int yycolumn;
-
-        /// <summary> zzAtBOL == true &lt;=&gt; the scanner is currently at the beginning of a line</summary>
-        private bool zzAtBOL = true;
-
-        /// <summary>zzAtEOF == true &lt;=&gt; the scanner is at the EOF </summary>
-        private bool zzAtEOF;
-        
-        /* user code: */
-        
-        public static readonly int ALPHANUM;
-        public static readonly int APOSTROPHE;
-        public static readonly int ACRONYM;
-        public static readonly int COMPANY;
-        public static readonly int EMAIL;
-        public static readonly int HOST;
-        public static readonly int NUM;
-        public static readonly int CJ;
-        /// <deprecated> this solves a bug where HOSTs that end with '.' are identified
-        /// as ACRONYMs.
-        /// </deprecated>
-        [Obsolete("this solves a bug where HOSTs that end with '.' are identified as ACRONYMs")]
-        public static readonly int ACRONYM_DEP;
-        
-        public static readonly System.String[] TOKEN_TYPES;
-        
-        public int Yychar()
-        {
-            return yychar;
-        }
-
-        /*
-        * Resets the Tokenizer to a new Reader.
-        */
-        internal void Reset(System.IO.TextReader r)
-        {
-            // reset to default buffer size, if buffer has grown
-            if (zzBuffer.Length > ZZ_BUFFERSIZE)
-            {
-                zzBuffer = new char[ZZ_BUFFERSIZE];
-            }
-            Yyreset(r);
-        }
-        
-        /// <summary> Fills Lucene token with the current token text.</summary>
-        internal void  GetText(Token t)
-        {
-            t.SetTermBuffer(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead);
-        }
-        
-        /// <summary> Fills TermAttribute with the current token text.</summary>
-        internal void  GetText(ITermAttribute t)
-        {
-            t.SetTermBuffer(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead);
-        }
-        
-        
-        /// <summary> Creates a new scanner
-        /// There is also a java.io.InputStream version of this constructor.
-        /// 
-        /// </summary>
-        /// <param name="in_Renamed"> the java.io.Reader to read input from.
-        /// </param>
-        internal StandardTokenizerImpl(System.IO.TextReader in_Renamed)
-        {
-            this.zzReader = in_Renamed;
-        }
-        
-        /// <summary> Creates a new scanner.
-        /// There is also java.io.Reader version of this constructor.
-        /// 
-        /// </summary>
-        /// <param name="in_Renamed"> the java.io.Inputstream to read input from.
-        /// </param>
-        internal StandardTokenizerImpl(System.IO.Stream in_Renamed):this(new System.IO.StreamReader(in_Renamed, System.Text.Encoding.Default))
-        {
-        }
-        
-        /// <summary> Unpacks the compressed character translation table.
-        /// 
-        /// </summary>
-        /// <param name="packed">  the packed character translation table
-        /// </param>
-        /// <returns>         the unpacked character translation table
-        /// </returns>
-        private static char[] ZzUnpackCMap(System.String packed)
-        {
-            char[] map = new char[0x10000];
-            int i = 0; /* index in packed string  */
-            int j = 0; /* index in unpacked array */
-            while (i < 1154)
-            {
-                int count = packed[i++];
-                char value_Renamed = packed[i++];
-                do 
-                    map[j++] = value_Renamed;
-                while (--count > 0);
-            }
-            return map;
-        }
-        
-        
-        /// <summary> Refills the input buffer.
-        /// </summary>
-        /// <returns><c>false</c>, iff there was new input.
-        /// 
-        /// </returns>
-        /// <exception cref="System.IO.IOException"> if any I/O-Error occurs
-        /// </exception>
-        private bool ZzRefill()
-        {
-            
-            /* first: make room (if you can) */
-            if (zzStartRead > 0)
-            {
-                Array.Copy(zzBuffer, zzStartRead, zzBuffer, 0, zzEndRead - zzStartRead);
-                
-                /* translate stored positions */
-                zzEndRead -= zzStartRead;
-                zzCurrentPos -= zzStartRead;
-                zzMarkedPos -= zzStartRead;
-                zzPushbackPos -= zzStartRead;
-                zzStartRead = 0;
-            }
-            
-            /* is the buffer big enough? */
-            if (zzCurrentPos >= zzBuffer.Length)
-            {
-                /* if not: blow it up */
-                char[] newBuffer = new char[zzCurrentPos * 2];
-                Array.Copy(zzBuffer, 0, newBuffer, 0, zzBuffer.Length);
-                zzBuffer = newBuffer;
-            }
-            
-            /* finally: fill the buffer with new input */
-            int numRead = zzReader.Read(zzBuffer, zzEndRead, zzBuffer.Length - zzEndRead);
-            
-            if (numRead <= 0)
-            {
-                return true;
-            }
-            else
-            {
-                zzEndRead += numRead;
-                return false;
-            }
-        }
-        
-        
-        /// <summary> Closes the input stream.</summary>
-        public void  Yyclose()
-        {
-            zzAtEOF = true; /* indicate end of file */
-            zzEndRead = zzStartRead; /* invalidate buffer    */
-            
-            if (zzReader != null)
-                zzReader.Close();
-        }
-        
-        
-        /// <summary> Resets the scanner to read from a new input stream.
-        /// Does not close the old reader.
-        /// 
-        /// All internal variables are reset, the old input stream 
-        /// <b>cannot</b> be reused (internal buffer is discarded and lost).
-        /// Lexical state is set to <tt>ZZ_INITIAL</tt>.
-        /// 
-        /// </summary>
-        /// <param name="reader">  the new input stream 
-        /// </param>
-        public void  Yyreset(System.IO.TextReader reader)
-        {
-            zzReader = reader;
-            zzAtBOL = true;
-            zzAtEOF = false;
-            zzEndRead = zzStartRead = 0;
-            zzCurrentPos = zzMarkedPos = zzPushbackPos = 0;
-            yyline = yychar = yycolumn = 0;
-            zzLexicalState = YYINITIAL;
-        }
-        
-        
-        /// <summary> Returns the current lexical state.</summary>
-        public int Yystate()
-        {
-            return zzLexicalState;
-        }
-        
-        
-        /// <summary> Enters a new lexical state
-        /// 
-        /// </summary>
-        /// <param name="newState">the new lexical state
-        /// </param>
-        public void  Yybegin(int newState)
-        {
-            zzLexicalState = newState;
-        }
-        
-        
-        /// <summary> Returns the text matched by the current regular expression.</summary>
-        public System.String Yytext()
-        {
-            return new System.String(zzBuffer, zzStartRead, zzMarkedPos - zzStartRead);
-        }
-        
-        
-        /// <summary> Returns the character at position <tt>pos</tt> from the 
-        /// matched text. 
-        /// 
-        /// It is equivalent to yytext().charAt(pos), but faster
-        /// 
-        /// </summary>
-        /// <param name="pos">the position of the character to fetch. 
-        /// A value from 0 to yylength()-1.
-        /// 
-        /// </param>
-        /// <returns> the character at position pos
-        /// </returns>
-        public char Yycharat(int pos)
-        {
-            return zzBuffer[zzStartRead + pos];
-        }
-        
-        
-        /// <summary> Returns the length of the matched text region.</summary>
-        public int Yylength()
-        {
-            return zzMarkedPos - zzStartRead;
-        }
-        
-        
-        /// <summary> Reports an error that occured while scanning.
-        /// 
-        /// In a wellformed scanner (no or only correct usage of 
-        /// yypushback(int) and a match-all fallback rule) this method 
-        /// will only be called with things that "Can't Possibly Happen".
-        /// If this method is called, something is seriously wrong
-        /// (e.g. a JFlex bug producing a faulty scanner etc.).
-        /// 
-        /// Usual syntax/scanner level error handling should be done
-        /// in error fallback rules.
-        /// 
-        /// </summary>
-        /// <param name="errorCode"> the code of the errormessage to display
-        /// </param>
-        private void  ZzScanError(int errorCode)
-        {
-            System.String message;
-            try
-            {
-                message = ZZ_ERROR_MSG[errorCode];
-            }
-            catch (System.IndexOutOfRangeException)
-            {
-                message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
-            }
-            
-            throw new System.ApplicationException(message);
-        }
-        
-        
-        /// <summary> Pushes the specified amount of characters back into the input stream.
-        /// 
-        /// They will be read again by then next call of the scanning method
-        /// 
-        /// </summary>
-        /// <param name="number"> the number of characters to be read again.
-        /// This number must not be greater than yylength()!
-        /// </param>
-        public virtual void  Yypushback(int number)
-        {
-            if (number > Yylength())
-                ZzScanError(ZZ_PUSHBACK_2BIG);
-            
-            zzMarkedPos -= number;
-        }
-        
-        
-        /// <summary> Resumes scanning until the next regular expression is matched,
-        /// the end of input is encountered or an I/O-Error occurs.
-        /// 
-        /// </summary>
-        /// <returns>      the next token
-        /// </returns>
-        /// <exception cref="System.IO.IOException"> if any I/O-Error occurs
-        /// </exception>
-        public virtual int GetNextToken()
-        {
-            int zzInput;
-            int zzAction;
-            
-            // cached fields:
-            int zzCurrentPosL;
-            int zzMarkedPosL;
-            int zzEndReadL = zzEndRead;
-            char[] zzBufferL = zzBuffer;
-            char[] zzCMapL = ZZ_CMAP;
-            
-            int[] zzTransL = ZZ_TRANS;
-            int[] zzRowMapL = ZZ_ROWMAP;
-            int[] zzAttrL = ZZ_ATTRIBUTE;
-            
-            while (true)
-            {
-                zzMarkedPosL = zzMarkedPos;
-                
-                yychar += zzMarkedPosL - zzStartRead;
-                
-                zzAction = - 1;
-                
-                zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
-                
-                zzState = zzLexicalState;
-                
-                
-                {
-                    while (true)
-                    {
-                        
-                        if (zzCurrentPosL < zzEndReadL)
-                            zzInput = zzBufferL[zzCurrentPosL++];
-                        else if (zzAtEOF)
-                        {
-                            zzInput = YYEOF;
-                            goto zzForAction_brk;   // {{Aroush-2.9}} this 'goto' maybe in the wrong place
-                        }
-                        else
-                        {
-                            // store back cached positions
-                            zzCurrentPos = zzCurrentPosL;
-                            zzMarkedPos = zzMarkedPosL;
-                            bool eof = ZzRefill();
-                            // get translated positions and possibly new buffer
-                            zzCurrentPosL = zzCurrentPos;
-                            zzMarkedPosL = zzMarkedPos;
-                            zzBufferL = zzBuffer;
-                            zzEndReadL = zzEndRead;
-                            if (eof)
-                            {
-                                zzInput = YYEOF;
-                                goto zzForAction_brk;   // {{Aroush-2.9}} this 'goto' maybe in the wrong place
-                            }
-                            else
-                            {
-                                zzInput = zzBufferL[zzCurrentPosL++];
-                            }
-                        }
-                        int zzNext = zzTransL[zzRowMapL[zzState] + zzCMapL[zzInput]];
-                        if (zzNext == - 1)
-                        {
-                            goto zzForAction_brk;   // {{Aroush-2.9}} this 'goto' maybe in the wrong place
-                        }
-                        zzState = zzNext;
-                        
-                        int zzAttributes = zzAttrL[zzState];
-                        if ((zzAttributes & 1) == 1)
-                        {
-                            zzAction = zzState;
-                            zzMarkedPosL = zzCurrentPosL;
-                            if ((zzAttributes & 8) == 8)
-                            {
-                                goto zzForAction_brk;   // {{Aroush-2.9}} this 'goto' maybe in the wrong place
-                            }
-                        }
-                    }
-                }
-
-zzForAction_brk: ;  // {{Aroush-2.9}} this 'lable' maybe in the wrong place
-                
-                
-                // store back cached position
-                zzMarkedPos = zzMarkedPosL;
-                
-                switch (zzAction < 0?zzAction:ZZ_ACTION[zzAction])
-                {
-                    
-                    case 4: 
-                    {
-                        return HOST;
-                    }
-                    
-                    case 11:  break;
-                    
-                    case 9: 
-                    {
-                        return ACRONYM;
-                    }
-                    
-                    case 12:  break;
-                    
-                    case 8: 
-                    {
-                        return ACRONYM_DEP;
-                    }
-                    
-                    case 13:  break;
-                    
-                    case 1: 
-                        {
-                            /* ignore */
-                        }
-                        goto case 14;
-                    
-                    case 14:  break;
-                    
-                    case 5: 
-                    {
-                        return NUM;
-                    }
-                    
-                    case 15:  break;
-                    
-                    case 3: 
-                    {
-                        return CJ;
-                    }
-                    
-                    case 16:  break;
-                    
-                    case 2: 
-                    {
-                        return ALPHANUM;
-                    }
-                    
-                    case 17:  break;
-                    
-                    case 7: 
-                    {
-                        return COMPANY;
-                    }
-                    
-                    case 18:  break;
-                    
-                    case 6: 
-                    {
-                        return APOSTROPHE;
-                    }
-                    
-                    case 19:  break;
-                    
-                    case 10: 
-                    {
-                        return EMAIL;
-                    }
-                    
-                    case 20:  break;
-                    
-                    default: 
-                        if (zzInput == YYEOF && zzStartRead == zzCurrentPos)
-                        {
-                            zzAtEOF = true;
-                            return YYEOF;
-                        }
-                        else
-                        {
-                            ZzScanError(ZZ_NO_MATCH);
-                        }
-                        break;
-                    
-                }
-            }
-        }
-        static StandardTokenizerImpl()
-        {
-            ALPHANUM = StandardTokenizer.ALPHANUM;
-            APOSTROPHE = StandardTokenizer.APOSTROPHE;
-            ACRONYM = StandardTokenizer.ACRONYM;
-            COMPANY = StandardTokenizer.COMPANY;
-            EMAIL = StandardTokenizer.EMAIL;
-            HOST = StandardTokenizer.HOST;
-            NUM = StandardTokenizer.NUM;
-            CJ = StandardTokenizer.CJ;
-            ACRONYM_DEP = StandardTokenizer.ACRONYM_DEP;
-            TOKEN_TYPES = StandardTokenizer.TOKEN_TYPES;
-        }
-    }
-}
\ No newline at end of file


Mime
View raw message