kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ale...@apache.org
Subject [3/4] kudu git commit: faststring: add shrink_to_fit()
Date Thu, 11 May 2017 18:23:11 GMT
faststring: add shrink_to_fit()

This adds a shrink_to_fit() method which reallocates the underlying buffer of a
faststring to match its current length. This is useful in the case where a
faststring acts as a long-lived buffer which occasionally gets large values,
but often contains small ones.

Change-Id: I0e437ff180fccd1957d252fb9a3551bb91ba7917
Reviewed-on: http://gerrit.cloudera.org:8080/6835
Reviewed-by: David Ribeiro Alves <davidralves@gmail.com>
Tested-by: Kudu Jenkins


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/dce25916
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/dce25916
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/dce25916

Branch: refs/heads/master
Commit: dce25916f428659072fd1e140a9081829956468e
Parents: 43fbfdf
Author: Todd Lipcon <todd@cloudera.com>
Authored: Tue May 9 14:30:41 2017 -0700
Committer: Todd Lipcon <todd@apache.org>
Committed: Thu May 11 16:25:10 2017 +0000

----------------------------------------------------------------------
 src/kudu/util/CMakeLists.txt     |  1 +
 src/kudu/util/faststring-test.cc | 60 +++++++++++++++++++++++++++++++++++
 src/kudu/util/faststring.cc      | 21 ++++++++++--
 src/kudu/util/faststring.h       | 20 ++++++++++--
 4 files changed, 96 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/dce25916/src/kudu/util/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/util/CMakeLists.txt b/src/kudu/util/CMakeLists.txt
index d9a37cd..e4fff0c 100644
--- a/src/kudu/util/CMakeLists.txt
+++ b/src/kudu/util/CMakeLists.txt
@@ -339,6 +339,7 @@ ADD_KUDU_TEST(env-test LABELS no_tsan)
 ADD_KUDU_TEST(env_util-test)
 ADD_KUDU_TEST(errno-test)
 ADD_KUDU_TEST(failure_detector-test)
+ADD_KUDU_TEST(faststring-test)
 ADD_KUDU_TEST(file_cache-test)
 ADD_KUDU_TEST(file_cache-stress-test RUN_SERIAL true)
 ADD_KUDU_TEST(flag_tags-test)

http://git-wip-us.apache.org/repos/asf/kudu/blob/dce25916/src/kudu/util/faststring-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/faststring-test.cc b/src/kudu/util/faststring-test.cc
new file mode 100644
index 0000000..c57cb41
--- /dev/null
+++ b/src/kudu/util/faststring-test.cc
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <algorithm>
+#include "kudu/util/faststring.h"
+#include "kudu/util/random.h"
+#include "kudu/util/random_util.h"
+#include "kudu/util/test_util.h"
+
+namespace kudu {
+class FaststringTest : public KuduTest {};
+
+TEST_F(FaststringTest, TestShrinkToFit_Empty) {
+  faststring s;
+  s.shrink_to_fit();
+  ASSERT_EQ(faststring::kInitialCapacity, s.capacity());
+}
+
+// Test that, if the string contents is shorter than the initial capacity
+// of the faststring, shrink_to_fit() leaves the string in the built-in
+// array.
+TEST_F(FaststringTest, TestShrinkToFit_SmallerThanInitialCapacity) {
+  faststring s;
+  s.append("hello");
+  s.shrink_to_fit();
+  ASSERT_EQ(faststring::kInitialCapacity, s.capacity());
+}
+
+TEST_F(FaststringTest, TestShrinkToFit_Random) {
+  Random r(GetRandomSeed32());
+  int kMaxSize = faststring::kInitialCapacity * 2;
+  std::unique_ptr<char[]> random_bytes(new char[kMaxSize]);
+  RandomString(random_bytes.get(), kMaxSize, &r);
+
+  faststring s;
+  for (int i = 0; i < 100; i++) {
+    int new_size = r.Uniform(kMaxSize);
+    s.resize(new_size);
+    memcpy(s.data(), random_bytes.get(), new_size);
+    s.shrink_to_fit();
+    ASSERT_EQ(0, memcmp(s.data(), random_bytes.get(), new_size));
+    ASSERT_EQ(std::max<int>(faststring::kInitialCapacity, new_size), s.capacity());
+  }
+}
+
+} // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/dce25916/src/kudu/util/faststring.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/faststring.cc b/src/kudu/util/faststring.cc
index cf5dbd1..a1cd26b 100644
--- a/src/kudu/util/faststring.cc
+++ b/src/kudu/util/faststring.cc
@@ -18,8 +18,7 @@
 #include "kudu/util/faststring.h"
 
 #include <glog/logging.h>
-
-#include "kudu/gutil/gscoped_ptr.h"
+#include <memory>
 
 namespace kudu {
 
@@ -38,7 +37,7 @@ void faststring::GrowByAtLeast(size_t count) {
 
 void faststring::GrowArray(size_t newcapacity) {
   DCHECK_GE(newcapacity, capacity_);
-  gscoped_array<uint8_t> newdata(new uint8_t[newcapacity]);
+  std::unique_ptr<uint8_t[]> newdata(new uint8_t[newcapacity]);
   if (len_ > 0) {
     memcpy(&newdata[0], &data_[0], len_);
   }
@@ -53,5 +52,21 @@ void faststring::GrowArray(size_t newcapacity) {
   ASAN_POISON_MEMORY_REGION(data_ + len_, capacity_ - len_);
 }
 
+void faststring::ShrinkToFitInternal() {
+  DCHECK_NE(data_, initial_data_);
+  if (len_ <= kInitialCapacity) {
+    ASAN_UNPOISON_MEMORY_REGION(initial_data_, len_);
+    memcpy(initial_data_, &data_[0], len_);
+    delete[] data_;
+    data_ = initial_data_;
+    capacity_ = kInitialCapacity;
+  } else {
+    std::unique_ptr<uint8_t[]> newdata(new uint8_t[len_]);
+    memcpy(&newdata[0], &data_[0], len_);
+    delete[] data_;
+    data_ = newdata.release();
+    capacity_ = len_;
+  }
+}
 
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/dce25916/src/kudu/util/faststring.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/faststring.h b/src/kudu/util/faststring.h
index 5bd003a..3d25c84 100644
--- a/src/kudu/util/faststring.h
+++ b/src/kudu/util/faststring.h
@@ -30,6 +30,10 @@ namespace kudu {
 // instead of memsetting to \0)
 class faststring {
  public:
+  enum {
+    kInitialCapacity = 32
+  };
+
   faststring() :
     data_(initial_data_),
     len_(0),
@@ -198,6 +202,18 @@ class faststring {
                 str.size());
   }
 
+  // Reallocates the internal storage to fit only the current data.
+  //
+  // This may revert to using internal storage if the current length is shorter than
+  // kInitialCapacity. Note that, in that case, after this call, capacity() will return
+  // a capacity larger than the data length.
+  //
+  // Any pointers within this instance are invalidated.
+  void shrink_to_fit() {
+    if (data_ == initial_data_ || capacity_ == len_) return;
+    ShrinkToFitInternal();
+  }
+
   // Return a copy of this string as a std::string.
   std::string ToString() const {
     return std::string(reinterpret_cast<const char *>(data()),
@@ -227,9 +243,7 @@ class faststring {
   // the current capacity.
   void GrowArray(size_t newcapacity);
 
-  enum {
-    kInitialCapacity = 32
-  };
+  void ShrinkToFitInternal();
 
   uint8_t* data_;
   uint8_t initial_data_[kInitialCapacity];


Mime
View raw message