usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [01/33] usergrid git commit: USERGRID-1044: support for multi-worker tests without end that can be audited
Date Tue, 20 Oct 2015 21:07:02 GMT
Repository: usergrid
Updated Branches:
  refs/heads/usergrid-1007-shiro-cache 341b47365 -> 2fdf664c6


USERGRID-1044: support for multi-worker tests without end
that can be audited


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a9927a3b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a9927a3b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a9927a3b

Branch: refs/heads/usergrid-1007-shiro-cache
Commit: a9927a3baa23bcd30dcacb4255af649c38ed3d90
Parents: 91fa489
Author: Mike Dunker <mdunker@apigee.com>
Authored: Tue Oct 6 14:24:40 2015 -0700
Committer: Mike Dunker <mdunker@apigee.com>
Committed: Tue Oct 6 14:24:40 2015 -0700

----------------------------------------------------------------------
 stack/loadtests/runDeleteEntities.sh            |  13 ++-
 stack/loadtests/runLoadEntitiesUnlimited.sh     | 101 +++++++++++++++++++
 .../datagenerators/FeederGenerator.scala        |   9 +-
 .../usergrid/enums/ConfigProperties.scala       |  11 +-
 .../apache/usergrid/helpers/Extractors.scala    |  10 ++
 .../scenarios/EntityCollectionScenarios.scala   |  27 +++--
 .../org/apache/usergrid/settings/Settings.scala |  94 ++++++++++++-----
 stack/loadtests/testConfig.sh                   |   2 +
 8 files changed, 227 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/runDeleteEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runDeleteEntities.sh b/stack/loadtests/runDeleteEntities.sh
index 2013e89..d0da19b 100755
--- a/stack/loadtests/runDeleteEntities.sh
+++ b/stack/loadtests/runDeleteEntities.sh
@@ -42,24 +42,27 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 
 die() { echo "$@" 1>&2 ; exit 1; }
 
-[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided.  Example is $0 RAMP_USERS
RAMP_TIME(seconds) [NUM_ENTITIES [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]"
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided.  Example is $0 RAMP_USERS
RAMP_TIME(seconds) ENTITY_PREFIX [NUM_ENTITIES [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]"
 
 RAMP_USERS="$1"
 RAMP_TIME="$2"
-[ "$#" -ge 3 ] && NUM_ENTITIES="$3"
-[ "$#" -ge 4 ] && ENTITY_SEED="$4"
-[ "$#" -ge 5 ] && ENTITY_WORKER_NUM="$5"
-[ "$#" -ge 6 ] && ENTITY_WORKER_COUNT="$6"
+ENTITY_PREFIX="$3"
+[ "$#" -ge 4 ] && NUM_ENTITIES="$4"
+[ "$#" -ge 5 ] && ENTITY_SEED="$5"
+[ "$#" -ge 6 ] && ENTITY_WORKER_NUM="$6"
+[ "$#" -ge 7 ] && ENTITY_WORKER_COUNT="$7"
 
 shift $#
 
 SCENARIO_TYPE=deleteEntities
+INTERLEAVED_WORKER_FEED=true
 
 #Compile everything
 mvn compile
 
 #Execute the test
 mvn gatling:execute \
+-DinterleavedWorkerFeed=${INTERLEAVED_WORKER_FEED} \
 -DbaseUrl=${URL} \
 -DadminUser=${ADMIN_USER}  \
 -DadminPassword=${ADMIN_PASSWORD}  \

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/runLoadEntitiesUnlimited.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runLoadEntitiesUnlimited.sh b/stack/loadtests/runLoadEntitiesUnlimited.sh
new file mode 100755
index 0000000..9222a57
--- /dev/null
+++ b/stack/loadtests/runLoadEntitiesUnlimited.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#               http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ENTITY_WORKER_NUM=  #may be overridden on command line
+#ENTITY_WORKER_COUNT=  #may be overridden on command line
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES= #ignored
+#SKIP_SETUP=
+#COLLECTION=
+#ENTITY_TYPE=
+#ENTITY_PREFIX=
+#ENTITY_SEED=  #may be overridden on command line
+#RETRY_COUNT=
+#ENTITY_PROGRESS_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
+#FLUSH_CSV
+
+die() { echo "$@" 1>&2 ; exit 1; }
+
+[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided.  Example is $0 RAMP_USERS
RAMP_TIME(seconds) [UUID_FILENAME [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]"
+
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+[ "$#" -ge 3 ] && UUID_FILENAME="$3"
+[ "$#" -ge 4 ] && ENTITY_SEED="$4"
+[ "$#" -ge 5 ] && ENTITY_WORKER_NUM="$5"
+[ "$#" -ge 6 ] && ENTITY_WORKER_COUNT="$6"
+
+shift $#
+
+SCENARIO_TYPE=loadEntities
+NEW_CSV_ON_FLUSH=true
+FLUSH_CSV=10000
+
+# don't load entities as part of setup (loading entities is the point of the test)
+LOAD_ENTITIES=false
+
+#Compile everything
+mvn compile
+
+#Execute the test
+mvn gatling:execute \
+-DflushCsv=${FLUSH_CSV} \
+-DunlimitedFeed=true \
+-DnewCsvOnFlush=${NEW_CSV_ON_FLUSH} \
+-DbaseUrl=${URL} \
+-DadminUser=${ADMIN_USER}  \
+-DadminPassword=${ADMIN_PASSWORD}  \
+-DentityWorkerNum=${ENTITY_WORKER_NUM} \
+-DentityWorkerCount=${ENTITY_WORKER_COUNT} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
+-DentityType=${ENTITY_TYPE} \
+-DentityPrefix=${ENTITY_PREFIX} \
+-DentitySeed=${ENTITY_SEED} \
+-DretryCount=${RETRY_COUNT} \
+-DentityProgressCount=${ENTITY_PROGRESS_COUNT} \
+-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC}    \
+-DconstantUsersDuration=${CONSTANT_USERS_DURATION}    \
+-DscenarioType=${SCENARIO_TYPE} \
+-DloadEntities=${LOAD_ENTITIES} \
+-DrampUsers=${RAMP_USERS}  \
+-DrampTime=${RAMP_TIME}  \
+-DuuidFilename=${UUID_FILENAME} \
+-DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
+-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
+

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index ca93fdc..e08f8e8 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -220,16 +220,19 @@ object FeederGenerator {
 
    override def next(): Map[String, String] = {
      val i = counter.getAndIncrement()
-     val seededVal = i + seed
+     val seededVal = if (Settings.interleavedWorkerFeed) seed + (i * Settings.entityWorkerCount)
+ (Settings.entityWorkerNum - 1) else i + seed
      val noPrefix = prefix == null || prefix == ""
      val entityName = if (noPrefix) seededVal.toString else prefix.concat(seededVal.toString)
      val entity = EntityDataGenerator.generateEntity(entityType, if (noPrefix) null else
entityName, seededVal)
+     //println(entity)
      val entityUrl = Settings.baseCollectionUrl + "/" + entityName
-     val validEntity = if (i >= numEntities) "no" else "yes"
+     val validEntity = if (!Settings.unlimitedFeed && i >= numEntities) "no" else
"yes"
+     val collectionName = Settings.app + "/" + Settings.collection
 
      // println(entityName)
 
-     Map("entityName" -> entityName, "entity" -> entity, "entityUrl" -> entityUrl,
"validEntity" -> validEntity, "entityNum" -> (i+1).toString, "seededEntityNum" ->
seededVal.toString)
+     Map("entityName" -> entityName, "entity" -> entity, "entityUrl" -> entityUrl,
"validEntity" -> validEntity, "entityNum" -> (i+1).toString, "seededEntityNum" ->
seededVal.toString,
+         "collectionName" -> collectionName)
    }
  }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
index df0b325..ba09979 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
@@ -81,6 +81,10 @@ object ConfigProperties {
   val EntityNumberProperty = "entityNumberProperty"
   val QueryParams = "queryParams"
   val CsvFeedPattern = "csvFeedPattern"
+  val UnlimitedFeed = "unlimitedFeed"
+  val FlushCsv = "flushCsv"
+  val InterleavedWorkerFeed = "interleavedWorkerFeed"
+  val NewCsvOnFlush = "newCsvOnFlush"
 
   val Values = Seq(Org,App,AdminUser,AdminPassword,BaseUrl,AuthType,TokenType,SkipSetup,CreateOrg,CreateApp,LoadEntities,
     ScenarioType,RampUsers,ConstantUsersPerSec,ConstantUsersDuration,UserSeed,AppUser,AppUserPassword,NumEntities,
@@ -89,7 +93,8 @@ object ConfigProperties {
     OrgCreationName,OrgCreationEmail,OrgCreationPassword,UpdateProperty,UpdateValue,EntityWorkerCount,EntityWorkerNum,
     UuidFilename,AuditUuidFilename,FailedUuidFilename,SandboxCollection,PurgeUsers,RetryCount,LaterThanTimestamp,
     EntityProgressCount,InjectionList,PrintFailedRequests,GetViaQuery,MultiPropertyPrefix,MultiPropertyCount,
-    MultiPropertySizeInK,EntityNumberProperty,QueryParams,CsvFeedPattern)
+    MultiPropertySizeInK,EntityNumberProperty,QueryParams,CsvFeedPattern,UnlimitedFeed,FlushCsv,InterleavedWorkerFeed,
+    NewCsvOnFlush)
 
   def isValid(str: String): Boolean = {
     Values.contains(str)
@@ -158,6 +163,10 @@ object ConfigProperties {
         case EntityNumberProperty => ""
         case QueryParams => ""
         case CsvFeedPattern => org.apache.usergrid.enums.CsvFeedPatternType.Random
+        case UnlimitedFeed => false
+        case FlushCsv => 0L
+        case InterleavedWorkerFeed => false
+        case NewCsvOnFlush => false
       }
     } else {
       null

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
index d93c430..8a18053 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
@@ -51,6 +51,16 @@ object Extractors {
   }
 
   /**
+   * Will extract the modified date from the create response.  If the modified field is not
present, -1 will be set
+   */
+  def extractCreateModified(saveAsName: String) = {
+    jsonPath("$.entities[0].modified").ofType[Long].transformOption(extract => {
+      //it may or may not be present.  If it is, save it, otherwise save it as -1
+      extract.orElse(Some(-1))
+    }).saveAs(saveAsName)
+  }
+
+  /**
    * Will extract the audit entities from the get collection response.
    */
   def extractAuditEntities(saveAsName: String) = {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index 0ab2cd6..1b15103 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -36,6 +36,7 @@ object EntityCollectionScenarios {
   //The value for the cursor
   val SessionVarCursor: String = "cursor"
   val SessionVarUuid: String = "createUuid"
+  val SessionVarModified: String = "createModified"
 
   def entityGetUrl(useCursor: Boolean): String = {
     val url = s"/${Settings.collection}?" +
@@ -208,9 +209,13 @@ object EntityCollectionScenarios {
         .headers(Headers.authToken)
         .body(StringBody("""${entity}"""))
         // 200 for success, 400 if already exists
-        .check(status.in(Seq(200)), extractCreateUuid(SessionVarUuid)))
+        .check(status.in(Seq(200)), extractCreateUuid(SessionVarUuid), extractCreateModified(SessionVarModified)))
         .exec(session => {
-          Settings.addUuid(session("seededEntityNum").as[String].toInt, session(SessionVarUuid).as[String])
+          val uuid = session(SessionVarUuid).as[String]
+          val entityName = session("entityName").as[String]
+          val modified = session(SessionVarModified).as[Long]
+          val collectionName = session("collectionName").as[String]
+          Settings.addUuid(uuid, collectionName, entityName, modified)
           session
         })
     }
@@ -294,9 +299,13 @@ object EntityCollectionScenarios {
         .get("/" + Settings.collection + "/${entityName}")
         .queryParamMap(Settings.queryParamMap)
         .headers(Headers.authAnonymous)
-        .check(status.is(200), extractCreateUuid(SessionVarUuid)))
+        .check(status.is(200), extractCreateUuid(SessionVarUuid), extractCreateModified(SessionVarModified)))
         .exec(session => {
-          Settings.addUuid(session("seededEntityNum").as[String].toInt, session(SessionVarUuid).as[String])
+          val uuid = session(SessionVarUuid).as[String]
+          val entityName = session("entityName").as[String]
+          val modified = session(SessionVarModified).as[Long]
+          val collectionName = session("collectionName").as[String]
+          Settings.addUuid(uuid, collectionName, entityName, modified)
           session
         })
     }
@@ -308,10 +317,14 @@ object EntityCollectionScenarios {
         .get("/" + Settings.collection + "/${entityName}")
         .queryParamMap(Settings.queryParamMap)
         .headers(Headers.authToken)
-        .check(status.is(200), extractCreateUuid(SessionVarUuid)))
+        .check(status.is(200), extractCreateUuid(SessionVarUuid), extractCreateModified(SessionVarModified)))
         .exec(session => {
-        Settings.addUuid(session("seededEntityNum").as[String].toInt, session(SessionVarUuid).as[String])
-        session
+          val uuid = session(SessionVarUuid).as[String]
+          val entityName = session("entityName").as[String]
+          val modified = session(SessionVarModified).as[Long]
+          val collectionName = session("collectionName").as[String]
+          Settings.addUuid(uuid, collectionName, entityName, modified)
+          session
       })
     }
   )

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index e9dd2f2..27c1d33 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -181,6 +181,11 @@ object Settings {
   private val queryParamConfig = initStrSetting(ConfigProperties.QueryParams)
   val queryParamMap: Map[String,String] = mapFromQueryParamConfigString(queryParamConfig)
   val csvFeedPattern = initStrSetting(ConfigProperties.CsvFeedPattern)
+  val flushCsv:Long = initLongSetting(ConfigProperties.FlushCsv)
+  val unlimitedFeed:Boolean = initBoolSetting(ConfigProperties.UnlimitedFeed)
+  // unlimited feed forces interleaved worker feed
+  val interleavedWorkerFeed:Boolean = if (unlimitedFeed) true else initBoolSetting(ConfigProperties.InterleavedWorkerFeed)
+  val newCsvOnFlush:Boolean = initBoolSetting(ConfigProperties.NewCsvOnFlush)
 
   val multiPropertyPrefix = initStrSetting(ConfigProperties.MultiPropertyPrefix)
   val multiPropertyCount:Int = initIntSetting(ConfigProperties.MultiPropertyCount)
@@ -204,8 +209,9 @@ object Settings {
   private val leftOver:Int = totalNumEntities % entityWorkerCount  // will be 0 if only one
worker
   private val extraEntity:Int = if (entityWorkerNum <= leftOver) 1 else 0
   private val zeroBasedWorkerNum:Int = entityWorkerNum - 1
-  val entitySeed:Int = overallEntitySeed + zeroBasedWorkerNum * entitiesPerWorkerFloor +
(if (extraEntity == 1) zeroBasedWorkerNum else leftOver)
-  val numEntities:Int = entitiesPerWorkerFloor + extraEntity
+  val entitySeed:Int = if (unlimitedFeed) overallEntitySeed else overallEntitySeed + zeroBasedWorkerNum
* entitiesPerWorkerFloor + (if (extraEntity == 1) zeroBasedWorkerNum else leftOver)
+  // numEntities is used for random name generation, must be >= 0 even if not used for
entity counting (as in unlimitedFeed=true)
+  val numEntities:Int = if (unlimitedFeed) 1000000000 else entitiesPerWorkerFloor + extraEntity
 
   // UUID log file, have to go through this because creating a csv feeder with an invalid
csv file fails at maven compile time
   private val dummyTestCsv = ConfigProperties.getDefault(ConfigProperties.UuidFilename).toString
@@ -281,18 +287,53 @@ object Settings {
 
   val purgeUsers:Int = initIntSetting(ConfigProperties.PurgeUsers)
 
-  private var uuidMap: Map[Int, String] = Map()
-  private var entityCounter: Long = 0
-  private var lastEntityCountPrinted: Long = 0
-  def addUuid(num: Int, uuid: String): Unit = {
+  val auditUuidsHeader = "collection,name,uuid,modified"
+  val uuidsHeader = "name,uuid"
+  case class AuditList(var collection: String, var entityName: String, var uuid: String,
var modified: Long)
+
+  //private var uuidMap: Map[Int, String] = Map()
+  private var uuidList: mutable.MutableList[AuditList] = mutable.MutableList[AuditList]()
+  private var entityCounter: Long = 0L
+  private var lastEntityCountPrinted: Long = 0L
+  private var flushCounter: Long = 0L
+  private var firstFlush: Boolean = true
+  private var numberFlushes: Long = 0L
+  private var uuidWriter: PrintWriter = null
+  def addUuid(uuid: String, collection: String, entityName: String, modified: Long): Unit
= {
     if (captureUuids) {
-      uuidMap.synchronized {
-        uuidMap += (num -> uuid)
-        entityCounter += 1
+      uuidList.synchronized {
+        uuidList += AuditList(collection, entityName, uuid, modified)
+        entityCounter += 1L
+        flushCounter += 1L
         if (logEntityProgress && (entityCounter >= lastEntityCountPrinted + entityProgressCount))
{
           println(s"Entity: $entityCounter")
           lastEntityCountPrinted = entityCounter
         }
+        if (flushCsv > 0 && flushCounter >= flushCsv) {
+          if (uuidWriter == null) {
+            uuidWriter = {
+              val fileWithSuffix = f"$captureUuidFilename.$numberFlushes%04d"
+              val fos = new FileOutputStream(if (newCsvOnFlush) fileWithSuffix else captureUuidFilename)
+              new PrintWriter(fos, false)
+            }
+          }
+          if (newCsvOnFlush || firstFlush) {
+            uuidWriter.println(auditUuidsHeader)
+          }
+          val sortedUuidList: List[AuditList] = uuidList.toList.sortBy(e => (e.collection,
e.entityName, e.modified))
+          sortedUuidList.foreach { e =>
+            uuidWriter.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified}")
+          }
+          uuidWriter.flush()
+          if (newCsvOnFlush) {
+            uuidWriter.close()
+            uuidWriter = null
+          }
+          flushCounter = 0L
+          numberFlushes += 1L
+          uuidList.clear()
+          firstFlush = false
+        }
       }
     }
     // println(s"UUID: ${name},${uuid}")
@@ -300,34 +341,39 @@ object Settings {
 
   def writeUuidsToFile(): Unit = {
     if (captureUuids) {
-      val writer = {
-        val fos = new FileOutputStream(captureUuidFilename)
-        new PrintWriter(fos, false)
+      if (uuidWriter == null) {
+        uuidWriter = {
+          val fileWithSuffix = f"$captureUuidFilename.$numberFlushes%04d"
+          val fos = new FileOutputStream(if (newCsvOnFlush) fileWithSuffix else captureUuidFilename)
+          new PrintWriter(fos, false)
+        }
       }
-      writer.println("name,uuid")
-      val uuidList: List[(Int, String)] = uuidMap.toList.sortBy(l => l._1)
-      uuidList.foreach { l =>
-        writer.println(s"${Settings.entityPrefix}${l._1},${l._2}")
+      if (newCsvOnFlush || firstFlush) {
+        uuidWriter.println(auditUuidsHeader)
       }
-      writer.flush()
-      writer.close()
+      val sortedUuidList: List[AuditList] = uuidList.toList.sortBy(e => (e.collection,
e.entityName, e.modified))
+      sortedUuidList.foreach { e =>
+        uuidWriter.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified}")
+      }
+      uuidWriter.flush()
+      uuidWriter.close()
+      numberFlushes += 1L
+      uuidList.clear()
+      firstFlush = false
     }
   }
 
 
-  val auditUuidsHeader = "collection,name,uuid,modified"
-
-  case class AuditList(var collection: String, var entityName: String, var uuid: String,
var modified: Long)
 
   // key: uuid, value: collection
-  private var auditEntityCounter: Long = 0
-  private var lastAuditEntityCountPrinted: Long = 0
+  private var auditEntityCounter: Long = 0L
+  private var lastAuditEntityCountPrinted: Long = 0L
   private var auditUuidList: mutable.MutableList[AuditList] = mutable.MutableList[AuditList]()
   def addAuditUuid(uuid: String, collection: String, entityName: String, modified: Long):
Unit = {
     if (captureAuditUuids) {
       auditUuidList.synchronized {
         auditUuidList += AuditList(collection, entityName, uuid, modified)
-        auditEntityCounter += 1
+        auditEntityCounter += 1L
         if (logEntityProgress && (auditEntityCounter >= lastAuditEntityCountPrinted
+ entityProgressCount)) {
           println(s"Entity: $auditEntityCounter")
           lastAuditEntityCountPrinted = auditEntityCounter

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/testConfig.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/testConfig.sh b/stack/loadtests/testConfig.sh
index 72b17c3..466d8c9 100755
--- a/stack/loadtests/testConfig.sh
+++ b/stack/loadtests/testConfig.sh
@@ -62,3 +62,5 @@ PRINT_FAILED_REQUESTS=true
 
 GET_VIA_QUERY=false
 QUERY_PARAMS=
+
+FLUSH_CSV=10000


Mime
View raw message