lucy-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mar...@apache.org
Subject [01/10] git commit: refs/heads/master - Eliminate MemoryPool from SortFieldWriter.
Date Fri, 04 Jul 2014 21:11:06 GMT
Repository: lucy
Updated Branches:
  refs/heads/master 5c0d6d9ba -> 7ec276695


Eliminate MemoryPool from SortFieldWriter.

SortFieldWriter used to allocate values from the MemoryPool, which was
also used to track memory usage.  Now it uses ordinary objects and
memory consumption is tracked in a simple Counter, so the MemoryPool is
no longer needed.


Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/78b35b7f
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/78b35b7f
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/78b35b7f

Branch: refs/heads/master
Commit: 78b35b7fac81ab9cc0f3557c35e720241d194c9d
Parents: 9504f79
Author: Marvin Humphrey <marvin@rectangular.com>
Authored: Wed Jul 2 17:50:46 2014 -0700
Committer: Marvin Humphrey <marvin@rectangular.com>
Committed: Fri Jul 4 14:10:06 2014 -0700

----------------------------------------------------------------------
 core/Lucy/Index/SortFieldWriter.c   | 22 ++++++++--------------
 core/Lucy/Index/SortFieldWriter.cfh |  5 ++---
 core/Lucy/Index/SortWriter.c        |  9 +++------
 core/Lucy/Index/SortWriter.cfh      |  1 -
 4 files changed, 13 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucy/blob/78b35b7f/core/Lucy/Index/SortFieldWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortFieldWriter.c b/core/Lucy/Index/SortFieldWriter.c
index 40c896a..fc350ba 100644
--- a/core/Lucy/Index/SortFieldWriter.c
+++ b/core/Lucy/Index/SortFieldWriter.c
@@ -36,7 +36,6 @@
 #include "Lucy/Store/InStream.h"
 #include "Lucy/Store/OutStream.h"
 #include "Clownfish/Util/Memory.h"
-#include "Lucy/Util/MemoryPool.h"
 #include "Clownfish/Util/SortUtils.h"
 
 // Prepare to read back a run.
@@ -67,13 +66,13 @@ SI_increase_to_word_multiple(int64_t amount) {
 SortFieldWriter*
 SortFieldWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
                     PolyReader *polyreader, String *field,
-                    MemoryPool *memory_pool, Counter *counter, size_t mem_thresh,
+                    Counter *counter, size_t mem_thresh,
                     OutStream *temp_ord_out, OutStream *temp_ix_out,
                     OutStream *temp_dat_out) {
     SortFieldWriter *self
         = (SortFieldWriter*)VTable_Make_Obj(SORTFIELDWRITER);
     return SortFieldWriter_init(self, schema, snapshot, segment, polyreader,
-                                field, memory_pool, counter, mem_thresh, temp_ord_out,
+                                field, counter, mem_thresh, temp_ord_out,
                                 temp_ix_out, temp_dat_out);
 }
 
@@ -81,7 +80,7 @@ SortFieldWriter*
 SortFieldWriter_init(SortFieldWriter *self, Schema *schema,
                      Snapshot *snapshot, Segment *segment,
                      PolyReader *polyreader, String *field,
-                     MemoryPool *memory_pool, Counter *counter, size_t mem_thresh,
+                     Counter *counter, size_t mem_thresh,
                      OutStream *temp_ord_out, OutStream *temp_ix_out,
                      OutStream *temp_dat_out) {
     // Init.
@@ -109,7 +108,6 @@ SortFieldWriter_init(SortFieldWriter *self, Schema *schema,
     ivars->snapshot     = (Snapshot*)INCREF(snapshot);
     ivars->segment      = (Segment*)INCREF(segment);
     ivars->polyreader   = (PolyReader*)INCREF(polyreader);
-    ivars->mem_pool     = (MemoryPool*)INCREF(memory_pool);
     ivars->counter      = (Counter*)INCREF(counter);
     ivars->temp_ord_out = (OutStream*)INCREF(temp_ord_out);
     ivars->temp_ix_out  = (OutStream*)INCREF(temp_ix_out);
@@ -142,10 +140,10 @@ SortFieldWriter_Clear_Buffer_IMP(SortFieldWriter *self) {
     SortFieldWriter_Clear_Buffer_t super_clear_buffer
         = SUPER_METHOD_PTR(SORTFIELDWRITER, LUCY_SortFieldWriter_Clear_Buffer);
     super_clear_buffer(self);
-    // Note that we have not called MemPool_Release_All() on our memory pool.
-    // This is because the pool is shared amongst multiple SortFieldWriters
+    // Note that we have not Reset() the Counter which tracks memory usage.
+    // This is because the counter is shared amongst multiple SortFieldWriters
     // which belong to a parent SortWriter; it is the responsibility of the
-    // parent SortWriter to release the memory pool once **all** of its child
+    // parent SortWriter to reset it once **all** of its child
     // SortFieldWriters have cleared their buffers.
 }
 
@@ -160,7 +158,6 @@ SortFieldWriter_Destroy_IMP(SortFieldWriter *self) {
     DECREF(ivars->segment);
     DECREF(ivars->polyreader);
     DECREF(ivars->type);
-    DECREF(ivars->mem_pool);
     DECREF(ivars->counter);
     DECREF(ivars->temp_ord_out);
     DECREF(ivars->temp_ix_out);
@@ -220,7 +217,7 @@ SortFieldWriter_Add_Segment_IMP(SortFieldWriter *self, SegReader *reader,
     SortFieldWriterIVARS *const ivars = SortFieldWriter_IVARS(self);
     SortFieldWriter *run
         = SortFieldWriter_new(ivars->schema, ivars->snapshot, ivars->segment,
-                              ivars->polyreader, ivars->field, ivars->mem_pool,
ivars->counter,
+                              ivars->polyreader, ivars->field, ivars->counter,
                               ivars->mem_thresh, NULL, NULL, NULL);
     SortFieldWriterIVARS *const run_ivars = SortFieldWriter_IVARS(run);
     run_ivars->sort_cache = (SortCache*)INCREF(sort_cache);
@@ -411,7 +408,7 @@ SortFieldWriter_Flush_IMP(SortFieldWriter *self) {
     SortFieldWriter_Sort_Buffer(self);
     SortFieldWriter *run
         = SortFieldWriter_new(ivars->schema, ivars->snapshot, ivars->segment,
-                              ivars->polyreader, ivars->field, ivars->mem_pool,
ivars->counter,
+                              ivars->polyreader, ivars->field, ivars->counter,
                               ivars->mem_thresh, NULL, NULL, NULL);
     SortFieldWriterIVARS *const run_ivars = SortFieldWriter_IVARS(run);
 
@@ -464,7 +461,6 @@ SortFieldWriter_Refill_IMP(SortFieldWriter *self) {
               buf_count);
     }
     SortFieldWriter_Clear_Buffer(self);
-    MemPool_Release_All(ivars->mem_pool);
     Counter_Reset(ivars->counter);
     S_lazy_init_sorted_ids(self);
 
@@ -683,9 +679,7 @@ S_flip_run(SortFieldWriter *run, size_t sub_thresh, InStream *ord_in,
     run_ivars->flipped = true;
 
     // Get our own slice of mem_thresh.
-    DECREF(run_ivars->mem_pool);
     DECREF(run_ivars->counter);
-    run_ivars->mem_pool   = MemPool_new(0);
     run_ivars->counter    = Counter_new();
     run_ivars->mem_thresh = sub_thresh;
 

http://git-wip-us.apache.org/repos/asf/lucy/blob/78b35b7f/core/Lucy/Index/SortFieldWriter.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortFieldWriter.cfh b/core/Lucy/Index/SortFieldWriter.cfh
index df82067..e2b996f 100644
--- a/core/Lucy/Index/SortFieldWriter.cfh
+++ b/core/Lucy/Index/SortFieldWriter.cfh
@@ -26,7 +26,6 @@ class Lucy::Index::SortFieldWriter
     PolyReader *polyreader;
     FieldType  *type;
     I32Array   *doc_map;
-    MemoryPool *mem_pool;
     Counter    *counter;
     int32_t     field_num;
     int32_t     null_ord;
@@ -55,14 +54,14 @@ class Lucy::Index::SortFieldWriter
 
     inert incremented SortFieldWriter*
     new(Schema *schema, Snapshot *snapshot, Segment *segment,
-        PolyReader *polyreader, String *field, MemoryPool *memory_pool, Counter *counter,
+        PolyReader *polyreader, String *field, Counter *counter,
         size_t mem_thresh, OutStream *temp_ord_out, OutStream *temp_ix_out,
         OutStream *temp_dat_out);
 
     inert SortFieldWriter*
     init(SortFieldWriter *self, Schema *schema, Snapshot *snapshot,
          Segment *segment, PolyReader *polyreader, String *field,
-         MemoryPool *memory_pool, Counter *counter, size_t mem_thresh, OutStream *temp_ord_out,
+         Counter *counter, size_t mem_thresh, OutStream *temp_ord_out,
          OutStream *temp_ix_out, OutStream *temp_dat_out);
 
     void

http://git-wip-us.apache.org/repos/asf/lucy/blob/78b35b7f/core/Lucy/Index/SortWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortWriter.c b/core/Lucy/Index/SortWriter.c
index fc323d0..bd3b690 100644
--- a/core/Lucy/Index/SortWriter.c
+++ b/core/Lucy/Index/SortWriter.c
@@ -34,7 +34,6 @@
 #include "Lucy/Store/InStream.h"
 #include "Lucy/Store/OutStream.h"
 #include "Clownfish/Util/Memory.h"
-#include "Lucy/Util/MemoryPool.h"
 #include "Clownfish/Util/SortUtils.h"
 
 int32_t SortWriter_current_file_format = 3;
@@ -63,7 +62,6 @@ SortWriter_init(SortWriter *self, Schema *schema, Snapshot *snapshot,
     ivars->temp_ord_out    = NULL;
     ivars->temp_ix_out     = NULL;
     ivars->temp_dat_out    = NULL;
-    ivars->mem_pool        = MemPool_new(0);
     ivars->counter         = Counter_new();
     ivars->mem_thresh      = default_mem_thresh;
     ivars->flush_at_finish = false;
@@ -81,7 +79,6 @@ SortWriter_Destroy_IMP(SortWriter *self) {
     DECREF(ivars->temp_ord_out);
     DECREF(ivars->temp_ix_out);
     DECREF(ivars->temp_dat_out);
-    DECREF(ivars->mem_pool);
     DECREF(ivars->counter);
     SUPER_DESTROY(self, SORTWRITER);
 }
@@ -126,7 +123,7 @@ S_lazy_init_field_writer(SortWriter *self, int32_t field_num) {
         String *field = Seg_Field_Name(ivars->segment, field_num);
         field_writer
             = SortFieldWriter_new(ivars->schema, ivars->snapshot, ivars->segment,
-                                  ivars->polyreader, field, ivars->mem_pool, ivars->counter,
+                                  ivars->polyreader, field, ivars->counter,
                                   ivars->mem_thresh, ivars->temp_ord_out,
                                   ivars->temp_ix_out, ivars->temp_dat_out);
         VA_Store(ivars->field_writers, field_num, (Obj*)field_writer);
@@ -152,14 +149,14 @@ SortWriter_Add_Inverted_Doc_IMP(SortWriter *self, Inverter *inverter,
     }
 
     // If our SortFieldWriters have collectively passed the memory threshold,
-    // flush all of them, then release all unique values with a single action.
+    // flush all of them, then reset the counter which tracks memory
+    // consumption.
     if (Counter_Get_Value(ivars->counter) > ivars->mem_thresh) {
         for (uint32_t i = 0; i < VA_Get_Size(ivars->field_writers); i++) {
             SortFieldWriter *const field_writer
                 = (SortFieldWriter*)VA_Fetch(ivars->field_writers, i);
             if (field_writer) { SortFieldWriter_Flush(field_writer); }
         }
-        MemPool_Release_All(ivars->mem_pool);
         Counter_Reset(ivars->counter);
         ivars->flush_at_finish = true;
     }

http://git-wip-us.apache.org/repos/asf/lucy/blob/78b35b7f/core/Lucy/Index/SortWriter.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortWriter.cfh b/core/Lucy/Index/SortWriter.cfh
index ba9bd35..6f4a242 100644
--- a/core/Lucy/Index/SortWriter.cfh
+++ b/core/Lucy/Index/SortWriter.cfh
@@ -35,7 +35,6 @@ class Lucy::Index::SortWriter inherits Lucy::Index::DataWriter {
     OutStream  *temp_ord_out;
     OutStream  *temp_ix_out;
     OutStream  *temp_dat_out;
-    MemoryPool *mem_pool;
     Counter    *counter;
     size_t      mem_thresh;
     bool        flush_at_finish;


Mime
View raw message