lucy-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mar...@apache.org
Subject [09/10] git commit: refs/heads/master - Improve mem tracking.
Date Fri, 04 Jul 2014 21:11:14 GMT
Improve mem tracking.

Cache size of value object.  Account for size of SFWriterElem.  Handle
blob fields better.


Project: http://git-wip-us.apache.org/repos/asf/lucy/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucy/commit/836d574f
Tree: http://git-wip-us.apache.org/repos/asf/lucy/tree/836d574f
Diff: http://git-wip-us.apache.org/repos/asf/lucy/diff/836d574f

Branch: refs/heads/master
Commit: 836d574f48ec19cddc82de7000d82bff6c139df0
Parents: ad101c8
Author: Marvin Humphrey <marvin@rectangular.com>
Authored: Thu Jul 3 12:54:33 2014 -0700
Committer: Marvin Humphrey <marvin@rectangular.com>
Committed: Fri Jul 4 14:10:07 2014 -0700

----------------------------------------------------------------------
 core/Lucy/Index/SortFieldWriter.c   | 48 +++++++++++++++++++-------------
 core/Lucy/Index/SortFieldWriter.cfh |  1 +
 2 files changed, 29 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucy/blob/836d574f/core/Lucy/Index/SortFieldWriter.c
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortFieldWriter.c b/core/Lucy/Index/SortFieldWriter.c
index fc350ba..a59ed3d 100644
--- a/core/Lucy/Index/SortFieldWriter.c
+++ b/core/Lucy/Index/SortFieldWriter.c
@@ -120,10 +120,17 @@ SortFieldWriter_init(SortFieldWriter *self, Schema *schema,
                           Schema_Fetch_Type(ivars->schema, field), FIELDTYPE);
     ivars->type    = (FieldType*)INCREF(type);
     ivars->prim_id = FType_Primitive_ID(type);
-    if (ivars->prim_id == FType_TEXT || ivars->prim_id == FType_BLOB) {
+    ivars->mem_per_entry = VTable_Get_Obj_Alloc_Size(SFWRITERELEM);
+    if (ivars->prim_id == FType_TEXT) {
+        ivars->mem_per_entry += VTable_Get_Obj_Alloc_Size(STRING);
+        ivars->var_width = true;
+    }
+    else if (ivars->prim_id == FType_BLOB) {
+        ivars->mem_per_entry += VTable_Get_Obj_Alloc_Size(BYTEBUF);
         ivars->var_width = true;
     }
     else {
+        ivars->mem_per_entry += VTable_Get_Obj_Alloc_Size(FLOAT64);
         ivars->var_width = false;
     }
     ivars->uniq_vals = Hash_new(0);
@@ -181,31 +188,32 @@ SortFieldWriter_Get_Ord_Width_IMP(SortFieldWriter *self) {
     return SortFieldWriter_IVARS(self)->ord_width;
 }
 
-static Obj*
-S_find_unique_value(Hash *uniq_vals, Counter *counter, Obj *val) {
-    int32_t  hash_sum  = Obj_Hash_Sum(val);
-    Obj     *uniq_val  = Hash_Find_Key(uniq_vals, val, hash_sum);
+void
+SortFieldWriter_Add_IMP(SortFieldWriter *self, int32_t doc_id, Obj *value) {
+    SortFieldWriterIVARS *const ivars = SortFieldWriter_IVARS(self);
+    Hash    *uniq_vals = ivars->uniq_vals;
+    Counter *counter   = ivars->counter;
+
+    // Uniq-ify the value.
+    int32_t  hash_sum  = Obj_Hash_Sum(value);
+    Obj     *uniq_val  = Hash_Find_Key(uniq_vals, value, hash_sum);
     if (!uniq_val) {
-        Hash_Store(uniq_vals, val, (Obj*)CFISH_TRUE);
-        VTable *vtable = Obj_Get_VTable(val);
-        Counter_Add(counter, VTable_Get_Obj_Alloc_Size(vtable));
-        if (vtable == STRING) {
-            int64_t size = Str_Get_Size((String*)val) + 1;
+        Hash_Store(uniq_vals, value, (Obj*)CFISH_TRUE);
+        Counter_Add(counter, ivars->mem_per_entry);
+        if (ivars->prim_id == FType_TEXT) {
+            int64_t size = Str_Get_Size((String*)value) + 1;
+            size = SI_increase_to_word_multiple(size);
+            Counter_Add(counter, size);
+        }
+        else if (ivars->prim_id == FType_BLOB) {
+            int64_t size = BB_Get_Size((ByteBuf*)value) + 1;
             size = SI_increase_to_word_multiple(size);
             Counter_Add(counter, size);
         }
-        uniq_val = Hash_Find_Key(uniq_vals, val, hash_sum);
+        uniq_val = Hash_Find_Key(uniq_vals, value, hash_sum);
     }
-    return uniq_val;
-}
-
-void
-SortFieldWriter_Add_IMP(SortFieldWriter *self, int32_t doc_id, Obj *value) {
-    SortFieldWriterIVARS *const ivars = SortFieldWriter_IVARS(self);
 
-    // Uniq-ify the value, and record it for this document.
-    Obj *copy = S_find_unique_value(ivars->uniq_vals, ivars->counter, value);
-    SFWriterElem *elem = S_SFWriterElem_create(copy, doc_id);
+    SFWriterElem *elem = S_SFWriterElem_create(uniq_val, doc_id);
     SortFieldWriter_Feed(self, (Obj*)elem);
     ivars->count++;
 }

http://git-wip-us.apache.org/repos/asf/lucy/blob/836d574f/core/Lucy/Index/SortFieldWriter.cfh
----------------------------------------------------------------------
diff --git a/core/Lucy/Index/SortFieldWriter.cfh b/core/Lucy/Index/SortFieldWriter.cfh
index 3625af3..22b2d74 100644
--- a/core/Lucy/Index/SortFieldWriter.cfh
+++ b/core/Lucy/Index/SortFieldWriter.cfh
@@ -29,6 +29,7 @@ class Lucy::Index::SortFieldWriter
     Counter    *counter;
     int32_t     field_num;
     int32_t     null_ord;
+    size_t      mem_per_entry;
     int8_t      prim_id;
     int32_t     count;
     OutStream  *temp_ord_out;


Mime
View raw message