usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From toddn...@apache.org
Subject [13/21] incubator-usergrid git commit: USERGRID-871: print settings summary + auto support multiple workers
Date Mon, 03 Aug 2015 20:30:57 GMT
USERGRID-871: print settings summary + auto support multiple workers


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bfc2be2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bfc2be2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bfc2be2b

Branch: refs/heads/two-dot-o-dev
Commit: bfc2be2b99e0892b5380310513abb5d6cc43736a
Parents: eec8175
Author: Mike Dunker <mdunker@apigee.com>
Authored: Mon Jul 27 16:30:56 2015 -0700
Committer: Mike Dunker <mdunker@apigee.com>
Committed: Mon Jul 27 16:30:56 2015 -0700

----------------------------------------------------------------------
 stack/loadtests/runLoadEntities.sh              |  8 ++-
 .../org/apache/usergrid/enums/AuthType.scala    |  1 -
 .../usergrid/enums/ConfigProperties.scala       |  4 +-
 .../apache/usergrid/helpers/Extractors.scala    | 10 ++++
 .../scenarios/EntityCollectionScenarios.scala   |  1 +
 .../org/apache/usergrid/settings/Settings.scala | 51 ++++++++++++++++++--
 .../simulations/ConfigurableSimulation.scala    |  4 ++
 7 files changed, 70 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bfc2be2b/stack/loadtests/runLoadEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runLoadEntities.sh b/stack/loadtests/runLoadEntities.sh
index d213fd4..e166c58 100755
--- a/stack/loadtests/runLoadEntities.sh
+++ b/stack/loadtests/runLoadEntities.sh
@@ -19,7 +19,7 @@ die() { echo "$@" 1>&2 ; exit 1; }
 #This is a script to simplify running gatling tests.  It will default several parameters,
invoke the maven plugins
 #Then aggregate the results
 ####
-[ "$#" -ge 6 ] || die "At least 6 arguments required, $# provided.  Example is $0 URL RAMP_USERS
RAMP_TIME(seconds) CONSTANT_USERS_PER_SEC, CONSTANT_USERS_DURATION(seconds) NUM_ENTITIES"
+[ "$#" -ge 8 ] || die "At least 8 arguments required, $# provided.  Example is $0 URL RAMP_USERS
RAMP_TIME(seconds) CONSTANT_USERS_PER_SEC, CONSTANT_USERS_DURATION(seconds) NUM_ENTITIES ENTITY_WORKER_NUM
ENTITY_WORKER_COUNT"
 
 URL="$1"
 RAMP_USERS="$2"
@@ -27,8 +27,10 @@ RAMP_TIME="$3"
 CONSTANT_USERS_PER_SEC="$4"
 CONSTANT_USERS_DURATION="$5"
 NUM_ENTITIES="$6"
+ENTITY_WORKER_NUM="$7"
+ENTITY_WORKER_COUNT="$8"
 
-shift 6
+shift 8
 
 #Compile everything
 mvn compile
@@ -85,6 +87,8 @@ mvn gatling:execute \
 -DendConditionType=${END_CONDITION_TYPE} \
 -DendMinutes=${END_MINUTES} \
 -DendRequestCount=${END_REQUEST_COUNT} \
+-DentityWorkerCount=${ENTITY_WORKER_COUNT} \
+-DentityWorkerNum=${ENTITY_WORKER_NUM} \
 -Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bfc2be2b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/AuthType.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/AuthType.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/AuthType.scala
index 6086fdd..abbb204 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/AuthType.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/AuthType.scala
@@ -9,7 +9,6 @@ object AuthType {
   val Basic = "basic"
 
   val Values = Seq(Anonymous,Token,Basic)
-
   def isValid(str: String): Boolean = {
     Values.contains(str)
   }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bfc2be2b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
index 2f13a12..855a7ae 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
@@ -46,12 +46,14 @@ object ConfigProperties {
   val OrgCreationPassword = "orgCreationPassword"
   val UpdateProperty = "updateProperty"
   val UpdateValue = "updateValue"
+  val EntityWorkerCount = "entityWorkerCount"
+  val EntityWorkerNum = "entityWorkerNum"
 
   val Values = Seq(Org,App,AdminUser,AdminPassword,BaseUrl,AuthType,TokenType,SkipSetup,CreateOrg,CreateApp,LoadEntities,
     ScenarioType,RampUsers,ConstantUsersPerSec,ConstantUsersDuration,UserSeed,AppUser,AppUserPassword,NumEntities,
     NumDevices,CollectionType,RampTime,Throttle,RpsTarget,RpsRampTime,HoldDuration,PushNotifier,EntityPrefix,EntityType,
     EntitySeed,SearchLimit,SearchQuery,EndConditionType,EndMinutes,EndRequestCount,OrgCreationUsername,OrgCreationName,
-    OrgCreationEmail,OrgCreationPassword,UpdateProperty,UpdateValue)
+    OrgCreationEmail,OrgCreationPassword,UpdateProperty,UpdateValue,EntityWorkerCount,EntityWorkerNum)
 
   def isValid(str: String): Boolean = {
     Values.contains(str)

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bfc2be2b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
index 35f75a7..f8aad98 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
@@ -41,6 +41,16 @@ object Extractors {
   }
 
   /**
+   * Will extract the uuid from the create response.  If the uuid is not present, an empty
string will be set
+   */
+  def extractUuid(saveAsName: String) = {
+    jsonPath("$.entities[0].uuid").transformOption(extract => {
+      //it may or may not be present.  If it is, save it, otherwise save it as an empty string
+      extract.orElse(Some(""))
+    }).saveAs(saveAsName)
+  }
+
+  /**
    * tries to extract the cursor from the session, if it exists, it returns true. if it's
the default, returns false
    * @param nameInSession The name of the variable in the session
    * @return

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bfc2be2b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index 1dd534c..1fa1c07 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -34,6 +34,7 @@ object EntityCollectionScenarios {
 
   //The value for the cursor
   val SessionVarCursor: String = "cursor"
+  val SessionVarUuid: String = "createUuid"
 
   def entityGetUrl(useCursor: Boolean): String = {
     val url = Settings.baseAppUrl + "/" + Settings.collectionType + "?dummy" +

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bfc2be2b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index a7669a1..cca3056 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -39,9 +39,9 @@ object Settings {
   val tokenType = System.getProperty(ConfigProperties.TokenType, TokenType.User)
 
   val skipSetup:Boolean = System.getProperty(ConfigProperties.SkipSetup) == "true"
-  val createOrg:Boolean = System.getProperty(ConfigProperties.CreateOrg) == "true"
-  val createApp:Boolean = System.getProperty(ConfigProperties.CreateApp) != "false"
-  val loadEntities:Boolean = System.getProperty(ConfigProperties.LoadEntities) != "false"
+  val createOrg:Boolean = !skipSetup && (System.getProperty(ConfigProperties.CreateOrg)
== "true")
+  val createApp:Boolean = !skipSetup && (System.getProperty(ConfigProperties.CreateApp)
!= "false")
+  val loadEntities:Boolean = !skipSetup && (System.getProperty(ConfigProperties.LoadEntities)
!= "false")
   val scenarioType = System.getProperty(ConfigProperties.ScenarioType, ScenarioType.NameRandomInfinite)
 
   val rampUsers:Int = Integer.getInteger(ConfigProperties.RampUsers, 0).toInt
@@ -53,7 +53,8 @@ object Settings {
   val appUserPassword = System.getProperty(ConfigProperties.AppUserPassword)
   val appUserBase64 = Base64.getEncoder.encodeToString((appUser + ":" + appUserPassword).getBytes(StandardCharsets.UTF_8))
 
-  val numEntities:Int = Integer.getInteger(ConfigProperties.NumEntities, 5000).toInt
+  var numEntities:Int = Integer.getInteger(ConfigProperties.NumEntities, 5000).toInt
+  val totalNumEntities:Int = numEntities
   val numDevices:Int = Integer.getInteger(ConfigProperties.NumDevices, 4000).toInt
 
   val collectionType = System.getProperty(ConfigProperties.CollectionType, "customentities")
@@ -78,7 +79,8 @@ object Settings {
   // Large Entity Collection settings
   val entityPrefix = System.getProperty(ConfigProperties.EntityPrefix, "entity")
   val entityType = System.getProperty(ConfigProperties.EntityType, EntityType.Basic) // basic/trivial/?
-  val entitySeed = Integer.getInteger(ConfigProperties.EntitySeed, 1).toInt
+  var entitySeed = Integer.getInteger(ConfigProperties.EntitySeed, 1).toInt
+  val overallEntitySeed = entitySeed
   val searchLimit = Integer.getInteger(ConfigProperties.SearchLimit, 0).toInt
   val searchQuery = System.getProperty(ConfigProperties.SearchQuery, "")
   val endConditionType = System.getProperty(ConfigProperties.EndConditionType, EndConditionType.MinutesElapsed)
@@ -96,6 +98,20 @@ object Settings {
   val updateValue = System.getProperty(ConfigProperties.UpdateValue, new Date().toString)
   val updateBody = Utils.toJSONStr(Map(updateProperty -> updateValue))
 
+  // Entity workers
+  val entityWorkerCount = Integer.getInteger(ConfigProperties.EntityWorkerCount,1)
+  val entityWorkerNum = Integer.getInteger(ConfigProperties.EntityWorkerNum, 1)
+
+  if (entityWorkerCount > 1 && entityWorkerNum >= 1 && entityWorkerNum
<= entityWorkerCount) {
+    // split entities across multiple workers
+    val entitiesPerWorkerFloor = totalNumEntities / entityWorkerCount
+    val leftOver = totalNumEntities % entityWorkerCount
+    val zeroBasedWorkerNum = entityWorkerNum - 1
+    val takeExtraEntity = if (entityWorkerNum <= leftOver) 1 else 0
+    entitySeed = overallEntitySeed + zeroBasedWorkerNum * entitiesPerWorkerFloor + (if (takeExtraEntity
== 1) zeroBasedWorkerNum else leftOver)
+    numEntities = entitiesPerWorkerFloor + takeExtraEntity
+  }
+
   def getUserFeeder():Array[Map[String, String]]= {
     FeederGenerator.generateUserWithGeolocationFeeder(totalUsers, userLocationRadius, centerLatitude,
centerLongitude)
   }
@@ -110,4 +126,29 @@ object Settings {
     testStartTime = System.currentTimeMillis()
   }
 
+  def printSettingsSummary(): Unit = {
+    val authTypeStr = authType + (if (authType == AuthType.Token) s"(${tokenType})" else
"")
+    val endConditionStr = if (endConditionType == EndConditionType.MinutesElapsed) s"${endMinutes}
minutes elapsed" else s"${endRequestCount} requests"
+    println("-----------------------------------------------------------------------------")
+    println("SIMULATION SETTINGS")
+    println("-----------------------------------------------------------------------------")
+    println()
+    println(s"Org:${org}  App:${app}  Collection:${collectionType}")
+    println(s"CreateOrg:${createOrg}  CreateApp:${createApp}  LoadEntities:${loadEntities}")
+    println(s"ScenarioType:${scenarioType}  AuthType:${authTypeStr}")
+    println()
+    println(s"Entity Type:${entityType}  Prefix:${entityPrefix}")
+    println()
+    println(s"Overall: NumEntities:${totalNumEntities}  Seed:${overallEntitySeed}  Workers:${entityWorkerCount}")
+    println(s"Worker:  NumEntities:${numEntities}  Seed:${entitySeed}  WorkerNum:${entityWorkerNum}")
+    println()
+    println(s"Ramp: Users:${rampUsers}  Time:${rampTime}")
+    println(s"Constant: UsersPerSec:${constantUsersPerSec}  Time:${constantUsersDuration}")
+    println(s"End Condition:${endConditionStr}")
+    println()
+    println("-----------------------------------------------------------------------------")
+  }
+
+  printSettingsSummary()
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bfc2be2b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
index 2e13a6c..8151cb1 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
@@ -70,5 +70,9 @@ class ConfigurableSimulation extends Simulation {
     println(s"scenarioType ${Settings.scenarioType} not found.")
   }
 
+  after(
+    Settings.printSettingsSummary()
+  )
+
 }
 


Mime
View raw message