lucy-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mar...@apache.org
Subject [lucy-commits] svn commit: r1183202 - in /incubator/lucy/branches/0.2: ./ core/Lucy/Search/ perl/t/
Date Fri, 14 Oct 2011 02:16:17 GMT
Author: marvin
Date: Fri Oct 14 02:16:16 2011
New Revision: 1183202

URL: http://svn.apache.org/viewvc?rev=1183202&view=rev
Log:
LUCY-180 Disable optimizations which skew scores.

Remove optimizations which change Matcher node structure per-segment,
resulting in skewed scores.  Harden ORMatcher and RequiredOptionalMatcher
against NULL children.

Modified:
    incubator/lucy/branches/0.2/   (props changed)
    incubator/lucy/branches/0.2/core/Lucy/Search/ORMatcher.c
    incubator/lucy/branches/0.2/core/Lucy/Search/ORQuery.c
    incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalMatcher.c
    incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalMatcher.cfh
    incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalQuery.c
    incubator/lucy/branches/0.2/perl/t/518-or_scorer.t
    incubator/lucy/branches/0.2/perl/t/519-req_opt_matcher.t
    incubator/lucy/branches/0.2/perl/t/527-req_opt_query.t

Propchange: incubator/lucy/branches/0.2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 14 02:16:16 2011
@@ -1 +1 @@
-/incubator/lucy/trunk:1148076,1148119,1159913,1164648,1171825,1183066,1183097,1183198
+/incubator/lucy/trunk:1148076,1148119,1159913,1164648,1171825,1183066,1183097,1183198-1183199

Modified: incubator/lucy/branches/0.2/core/Lucy/Search/ORMatcher.c
URL: http://svn.apache.org/viewvc/incubator/lucy/branches/0.2/core/Lucy/Search/ORMatcher.c?rev=1183202&r1=1183201&r2=1183202&view=diff
==============================================================================
--- incubator/lucy/branches/0.2/core/Lucy/Search/ORMatcher.c (original)
+++ incubator/lucy/branches/0.2/core/Lucy/Search/ORMatcher.c Fri Oct 14 02:16:16 2011
@@ -97,7 +97,9 @@ S_ormatcher_init2(ORMatcher *self, VArra
     // Prime queue.
     for (i = 0; i < self->max_size; i++) {
         Matcher *matcher = (Matcher*)VA_Fetch(children, i);
-        S_add_element(self, (Matcher*)INCREF(matcher), 0);
+        if (matcher) {
+            S_add_element(self, (Matcher*)INCREF(matcher), 0);
+        }
     }
 
     return self;

Modified: incubator/lucy/branches/0.2/core/Lucy/Search/ORQuery.c
URL: http://svn.apache.org/viewvc/incubator/lucy/branches/0.2/core/Lucy/Search/ORQuery.c?rev=1183202&r1=1183201&r2=1183202&view=diff
==============================================================================
--- incubator/lucy/branches/0.2/core/Lucy/Search/ORQuery.c (original)
+++ incubator/lucy/branches/0.2/core/Lucy/Search/ORQuery.c Fri Oct 14 02:16:16 2011
@@ -99,6 +99,7 @@ ORCompiler_make_matcher(ORCompiler *self
     uint32_t num_kids = VA_Get_Size(self->children);
 
     if (num_kids == 1) {
+        // No need for an ORMatcher wrapper.
         Compiler *only_child = (Compiler*)VA_Fetch(self->children, 0);
         return Compiler_Make_Matcher(only_child, reader, need_score);
     }
@@ -112,8 +113,8 @@ ORCompiler_make_matcher(ORCompiler *self
             Compiler *child = (Compiler*)VA_Fetch(self->children, i);
             Matcher *submatcher
                 = Compiler_Make_Matcher(child, reader, need_score);
+            VA_Push(submatchers, (Obj*)submatcher);
             if (submatcher != NULL) {
-                VA_Push(submatchers, (Obj*)submatcher);
                 num_submatchers++;
             }
         }
@@ -123,12 +124,6 @@ ORCompiler_make_matcher(ORCompiler *self
             DECREF(submatchers);
             return NULL;
         }
-        else if (num_submatchers == 1) {
-            // Only one submatcher, so no need for ORScorer wrapper.
-            Matcher *submatcher = (Matcher*)INCREF(VA_Fetch(submatchers, 0));
-            DECREF(submatchers);
-            return submatcher;
-        }
         else {
             Similarity *sim    = ORCompiler_Get_Similarity(self);
             Matcher    *retval = need_score

Modified: incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalMatcher.c
URL: http://svn.apache.org/viewvc/incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalMatcher.c?rev=1183202&r1=1183201&r2=1183202&view=diff
==============================================================================
--- incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalMatcher.c (original)
+++ incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalMatcher.c Fri Oct 14 02:16:16
2011
@@ -76,14 +76,15 @@ ReqOptMatcher_score(RequiredOptionalMatc
 
     if (self->opt_matcher_first_time) {
         self->opt_matcher_first_time = false;
-        if (!Matcher_Advance(self->opt_matcher, current_doc)) {
+        if (self->opt_matcher != NULL
+            && !Matcher_Advance(self->opt_matcher, current_doc)) {
             DECREF(self->opt_matcher);
             self->opt_matcher = NULL;
         }
     }
 
     if (self->opt_matcher == NULL) {
-        return Matcher_Score(self->req_matcher);
+        return Matcher_Score(self->req_matcher) * self->coord_factors[1];
     }
     else {
         int32_t opt_matcher_doc = Matcher_Get_Doc_ID(self->opt_matcher);
@@ -93,7 +94,8 @@ ReqOptMatcher_score(RequiredOptionalMatc
             if (!opt_matcher_doc) {
                 DECREF(self->opt_matcher);
                 self->opt_matcher = NULL;
-                return Matcher_Score(self->req_matcher);
+                float req_score = Matcher_Score(self->req_matcher);
+                return req_score * self->coord_factors[1];
             }
         }
 
@@ -104,7 +106,7 @@ ReqOptMatcher_score(RequiredOptionalMatc
             return score;
         }
         else {
-            return Matcher_Score(self->req_matcher);
+            return Matcher_Score(self->req_matcher) * self->coord_factors[1];
         }
     }
 }

Modified: incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalMatcher.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalMatcher.cfh?rev=1183202&r1=1183201&r2=1183202&view=diff
==============================================================================
--- incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalMatcher.cfh (original)
+++ incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalMatcher.cfh Fri Oct 14 02:16:16
2011
@@ -28,11 +28,11 @@ class Lucy::Search::RequiredOptionalMatc
 
     inert incremented RequiredOptionalMatcher*
     new(Similarity *similarity, Matcher *required_matcher,
-        Matcher *optional_matcher);
+        Matcher *optional_matcher = NULL);
 
     inert RequiredOptionalMatcher*
     init(RequiredOptionalMatcher *self, Similarity *similarity,
-         Matcher *required_matcher, Matcher *optional_matcher);
+         Matcher *required_matcher, Matcher *optional_matcher = NULL);
 
     public void
     Destroy(RequiredOptionalMatcher *self);

Modified: incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalQuery.c
URL: http://svn.apache.org/viewvc/incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalQuery.c?rev=1183202&r1=1183201&r2=1183202&view=diff
==============================================================================
--- incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalQuery.c (original)
+++ incubator/lucy/branches/0.2/core/Lucy/Search/RequiredOptionalQuery.c Fri Oct 14 02:16:16
2011
@@ -128,9 +128,6 @@ ReqOptCompiler_make_matcher(RequiredOpti
         DECREF(opt_matcher);
         return NULL;
     }
-    else if (opt_matcher == NULL) {
-        return req_matcher;
-    }
     else {
         Matcher *retval
             = (Matcher*)ReqOptMatcher_new(sim, req_matcher, opt_matcher);

Modified: incubator/lucy/branches/0.2/perl/t/518-or_scorer.t
URL: http://svn.apache.org/viewvc/incubator/lucy/branches/0.2/perl/t/518-or_scorer.t?rev=1183202&r1=1183201&r2=1183202&view=diff
==============================================================================
--- incubator/lucy/branches/0.2/perl/t/518-or_scorer.t (original)
+++ incubator/lucy/branches/0.2/perl/t/518-or_scorer.t Fri Oct 14 02:16:16 2011
@@ -17,17 +17,17 @@ use strict;
 use warnings;
 use lib 'buildlib';
 
-use Test::More tests => 900;
+use Test::More tests => 1617; 
 use Lucy::Test;
 use LucyX::Search::MockMatcher;
 use Lucy::Test::TestUtils qw( modulo_set doc_ids_from_td_coll );
 
 my $sim = Lucy::Index::Similarity->new;
 
-for my $interval_a ( 1 .. 10 ) {
-    for my $interval_b ( 5 .. 10 ) {
+for my $interval_a ( 1 .. 10, 1000 ) {
+    for my $interval_b ( 5 .. 10, 1000 ) {
         check_matcher( $interval_a, $interval_b );
-        for my $interval_c ( 30, 75 ) {
+        for my $interval_c ( 30, 75, 1000 ) {
             check_matcher( $interval_a, $interval_b, $interval_c );
             check_matcher( $interval_c, $interval_b, $interval_a );
         }
@@ -40,11 +40,16 @@ sub check_matcher {
     my $child_matchers
         = Lucy::Object::VArray->new( capacity => scalar @intervals );
     for my $doc_id_array (@doc_id_arrays) {
-        my $mock = LucyX::Search::MockMatcher->new(
-            doc_ids => $doc_id_array,
-            scores  => [ (1) x scalar @$doc_id_array ],
-        );
-        $child_matchers->push($mock);
+        if (@$doc_id_array) {
+            my $mock = LucyX::Search::MockMatcher->new(
+                doc_ids => $doc_id_array,
+                scores  => [ (1) x scalar @$doc_id_array ],
+            );
+            $child_matchers->push($mock);
+        }
+        else {
+            $child_matchers->push(undef);
+        }
     }
 
     my $or_scorer = Lucy::Search::ORScorer->new(

Modified: incubator/lucy/branches/0.2/perl/t/519-req_opt_matcher.t
URL: http://svn.apache.org/viewvc/incubator/lucy/branches/0.2/perl/t/519-req_opt_matcher.t?rev=1183202&r1=1183201&r2=1183202&view=diff
==============================================================================
--- incubator/lucy/branches/0.2/perl/t/519-req_opt_matcher.t (original)
+++ incubator/lucy/branches/0.2/perl/t/519-req_opt_matcher.t Fri Oct 14 02:16:16 2011
@@ -17,7 +17,7 @@ use strict;
 use warnings;
 use lib 'buildlib';
 
-use Test::More tests => 726;
+use Test::More tests => 792;
 use Lucy::Test;
 use LucyX::Search::MockMatcher;
 use Lucy::Test::TestUtils qw( modulo_set doc_ids_from_td_coll );
@@ -25,7 +25,7 @@ use Lucy::Test::TestUtils qw( modulo_set
 my $sim = Lucy::Index::Similarity->new;
 
 for my $req_interval ( 1 .. 10, 75 ) {
-    for my $opt_interval ( 1 .. 10, 75 ) {
+    for my $opt_interval ( 1 .. 10, 75, 1000 ) { # 1000 = no matches
         check_matcher( $req_interval, $opt_interval );
         check_matcher( $opt_interval, $req_interval );
     }
@@ -39,10 +39,13 @@ sub check_matcher {
         doc_ids => $req_docs,
         scores  => [ (1) x scalar @$req_docs ],
     );
-    my $opt_mock = LucyX::Search::MockMatcher->new(
-        doc_ids => $opt_docs,
-        scores  => [ (1) x scalar @$opt_docs ],
-    );
+    my $opt_mock;
+    if (@$opt_docs) {
+        $opt_mock = LucyX::Search::MockMatcher->new(
+            doc_ids => $opt_docs,
+            scores  => [ (1) x scalar @$opt_docs ],
+        );
+    }
     my $req_opt_matcher = Lucy::Search::RequiredOptionalMatcher->new(
         similarity       => $sim,
         required_matcher => $req_mock,

Modified: incubator/lucy/branches/0.2/perl/t/527-req_opt_query.t
URL: http://svn.apache.org/viewvc/incubator/lucy/branches/0.2/perl/t/527-req_opt_query.t?rev=1183202&r1=1183201&r2=1183202&view=diff
==============================================================================
--- incubator/lucy/branches/0.2/perl/t/527-req_opt_query.t (original)
+++ incubator/lucy/branches/0.2/perl/t/527-req_opt_query.t Fri Oct 14 02:16:16 2011
@@ -17,7 +17,7 @@ use strict;
 use warnings;
 use lib 'buildlib';
 
-use Test::More tests => 7;
+use Test::More tests => 6;
 use Storable qw( freeze thaw );
 use Lucy::Test::TestUtils qw( create_index );
 
@@ -52,15 +52,6 @@ my $matcher = $compiler->make_matcher( r
 isa_ok( $matcher, 'Lucy::Search::RequiredOptionalMatcher' );
 
 $req_opt_query = Lucy::Search::RequiredOptionalQuery->new(
-    required_query => $b_query,
-    optional_query => $x_query,
-);
-$matcher = $req_opt_query->make_compiler( searcher => $searcher )
-    ->make_matcher( reader => $reader, need_score => 0 );
-isa_ok( $matcher, 'Lucy::Search::TermMatcher',
-    "return required matcher only when opt matcher doesn't match" );
-
-$req_opt_query = Lucy::Search::RequiredOptionalQuery->new(
     required_query => $x_query,
     optional_query => $b_query,
 );



Mime
View raw message