lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nightowl...@apache.org
Subject [03/10] lucenenet git commit: Added test for compatibility with Lucene 4.8.0 index that has binary doc values.
Date Tue, 16 May 2017 14:28:23 GMT
Added test for compatibility with Lucene 4.8.0 index that has binary doc values.


Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/afb23de6
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/afb23de6
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/afb23de6

Branch: refs/heads/master
Commit: afb23de6b6c46f0a9f0a58c86aa07a1c7eb3b6f2
Parents: 0cc3cfe
Author: Shad Storhaug <shad@shadstorhaug.com>
Authored: Tue May 16 17:02:35 2017 +0700
Committer: Shad Storhaug <shad@shadstorhaug.com>
Committed: Tue May 16 19:17:11 2017 +0700

----------------------------------------------------------------------
 .../Index/TestBinaryDocValuesUpdates.cs         |  89 +++++++++++++++++++
 .../Index/index.48.w-binary-doc-values.zip      | Bin 0 -> 39790 bytes
 .../Index/taxo.48.w-binary-doc-values.zip       | Bin 0 -> 4869 bytes
 src/Lucene.Net.Tests/Lucene.Net.Tests.csproj    |   2 +
 src/Lucene.Net.Tests/project.json               |   2 +
 5 files changed, 93 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs b/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs
index 7f6a817..3b5e738 100644
--- a/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs
+++ b/src/Lucene.Net.Tests/Index/TestBinaryDocValuesUpdates.cs
@@ -1,13 +1,19 @@
 using Lucene.Net.Attributes;
 using Lucene.Net.Codecs;
 using Lucene.Net.Documents;
+using Lucene.Net.Facet;
+using Lucene.Net.Facet.Taxonomy;
+using Lucene.Net.Facet.Taxonomy.Directory;
 using Lucene.Net.Randomized.Generators;
+using Lucene.Net.Search;
+using Lucene.Net.Store;
 using Lucene.Net.Support;
 using Lucene.Net.Support.Threading;
 using NUnit.Framework;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Reflection;
 using System.Threading;
 
 namespace Lucene.Net.Index
@@ -1776,5 +1782,88 @@ namespace Lucene.Net.Index
 
             dir.Dispose();
         }
+
+        /// <summary>
+        /// Using the exact translation from Lucene, we had an issue where loss of float
precision was causing
+        /// index corruption when using binary doc values in combination with a 32 bit application
to write the index. 
+        /// Consequently, a 64 bit application could not read an index generated by a 32
bit application and a 
+        /// 32 bit application could not read an index written by Java Lucene (regardless
of bitness).
+        /// <para/>
+        /// This test is to verify that the current test environment (be it 32 or 64 bit)
can read an index with
+        /// binary doc values that was written by Lucene in Java.
+        /// <para/>
+        /// To be thorough, we should setup the testing to run both in 32 bit and 64 bit
on each platform. It would
+        /// be better if we could somehow generate the file under one bitness and consume
it under the other, but
+        /// that would be difficult to set up. So, we are just reading a known good index
and if we cannot read it
+        /// we know there is a problem. If the reading is fixed and the writing is different,
it will fail several tests anyway.
+        /// </summary>
+        [Test]
+        [LuceneNetSpecific]
+        public virtual void TestReadIndexBitness()
+        {
+            string[] alphabet = new string[] { "A", "B", "C", "D", "E", "F", "G", "H", "I",
"J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z" };
+            int iterations = 500;
+            FacetsConfig facetConfig = GetFacetConfig();
+
+            // Unzip index
+            DirectoryInfo indexDir = CreateTempDir("index");
+            using (Stream zipFileStream = this.GetType().GetTypeInfo().Assembly.FindAndGetManifestResourceStream(GetType(),
"index.48.w-binary-doc-values.zip"))
+            {
+                TestUtil.Unzip(zipFileStream, indexDir);
+            }
+
+            // Unzip taxonomy
+            DirectoryInfo taxoDir = CreateTempDir("taxo");
+            using (Stream zipFileStream = this.GetType().GetTypeInfo().Assembly.FindAndGetManifestResourceStream(GetType(),
"taxo.48.w-binary-doc-values.zip"))
+            {
+                TestUtil.Unzip(zipFileStream, taxoDir);
+            }
+
+
+            SimpleFSDirectory nFsd = new SimpleFSDirectory(indexDir);
+            IndexReader indexReader = DirectoryReader.Open(nFsd);
+            IndexSearcher searcher = new IndexSearcher(indexReader);
+
+            DirectoryTaxonomyReader taxoReader =
+                new DirectoryTaxonomyReader(FSDirectory.Open(taxoDir));
+
+            try
+            {
+                foreach (string letter in alphabet)
+                {
+                    FacetsCollector c = new FacetsCollector();
+                    searcher.Search(new TermQuery(new Term("Field1", letter)), c);
+
+                    Facets facets = new FastTaxonomyFacetCounts(taxoReader, facetConfig,
c);
+                    FacetResult result = facets.GetTopChildren(int.MaxValue, "facetField1");
+
+                    assertEquals(iterations, System.Convert.ToInt32(result.LabelValues[0].Value));
+
+                    FacetResult result2 = facets.GetTopChildren(int.MaxValue, "facetField2");
+
+                    
+                    for (int i = 0; i < iterations; i++)
+                    {
+                        assertEquals(i, System.Convert.ToInt32(result2.LabelValues[i].Label));
+                    }
+                }
+
+                System.Console.WriteLine("Success");
+            }
+            finally
+            {
+                IOUtils.Close(indexReader, taxoReader, nFsd);
+            }
+        }
+
+        private FacetsConfig GetFacetConfig()
+        {
+            FacetsConfig facetConfig = new FacetsConfig();
+            FacetsConfig config = new FacetsConfig();
+
+            config.SetRequireDimCount("facetField1", true);
+            config.SetRequireDimCount("facetField2", true);
+            return facetConfig;
+        }
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/Index/index.48.w-binary-doc-values.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.48.w-binary-doc-values.zip b/src/Lucene.Net.Tests/Index/index.48.w-binary-doc-values.zip
new file mode 100644
index 0000000..c18e39e
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.48.w-binary-doc-values.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/Index/taxo.48.w-binary-doc-values.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/taxo.48.w-binary-doc-values.zip b/src/Lucene.Net.Tests/Index/taxo.48.w-binary-doc-values.zip
new file mode 100644
index 0000000..413cb54
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/taxo.48.w-binary-doc-values.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
index bc1008b..a913b37 100644
--- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
+++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
@@ -113,6 +113,8 @@
     <EmbeddedResource Include="Index\unsupported.24.nocfs.zip" />
     <EmbeddedResource Include="Index\unsupported.29.cfs.zip" />
     <EmbeddedResource Include="Index\unsupported.29.nocfs.zip" />
+    <EmbeddedResource Include="Index\index.48.w-binary-doc-values.zip" />
+    <EmbeddedResource Include="Index\taxo.48.w-binary-doc-values.zip" />
     <None Include="Lucene.Net.snk" />
     <None Include="Lucene.Net.Tests.project.json" />
   </ItemGroup>

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/afb23de6/src/Lucene.Net.Tests/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/project.json b/src/Lucene.Net.Tests/project.json
index c8b25ae..b1cf1a4 100644
--- a/src/Lucene.Net.Tests/project.json
+++ b/src/Lucene.Net.Tests/project.json
@@ -33,7 +33,9 @@
         "Index/index.45.nocfs.zip",
         "Index/index.461.cfs.zip",
         "Index/index.461.nocfs.zip",
+        "Index/index.48.w-binary-doc-values.zip",
         "Index/moreterms.40.zip",
+        "Index/taxo.48.w-binary-doc-values.zip",
         "Index/unsupported.19.cfs.zip",
         "Index/unsupported.19.nocfs.zip",
         "Index/unsupported.20.cfs.zip",


Mime
View raw message