usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [2/6] usergrid git commit: USERGRID-871: Allow circular access for CSV feeder, plus readme fixes
Date Tue, 01 Sep 2015 15:39:18 GMT
USERGRID-871: Allow circular access for CSV feeder, plus readme fixes


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/61bc825f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/61bc825f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/61bc825f

Branch: refs/heads/two-dot-o-dev
Commit: 61bc825f57ad6300b8266bc5e067ae77e6aaa8d5
Parents: d626f8c
Author: Mike Dunker <mikedunker@calbears.net>
Authored: Mon Aug 31 17:09:01 2015 -0700
Committer: Mike Dunker <mikedunker@calbears.net>
Committed: Mon Aug 31 17:09:01 2015 -0700

----------------------------------------------------------------------
 stack/loadtests/README.md                       | 13 ++-
 stack/loadtests/runGetEntitiesByUuid.sh         | 92 ++++++++++++++++++++
 stack/loadtests/runRandomEntityByUuidTest.sh    | 87 ------------------
 .../usergrid/enums/ConfigProperties.scala       |  4 +-
 .../usergrid/enums/CsvFeedPatternType.scala     | 30 +++++++
 .../scenarios/EntityCollectionScenarios.scala   | 11 ++-
 .../org/apache/usergrid/settings/Settings.scala |  7 +-
 7 files changed, 147 insertions(+), 97 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/61bc825f/stack/loadtests/README.md
----------------------------------------------------------------------
diff --git a/stack/loadtests/README.md b/stack/loadtests/README.md
index 74f93bf..802c286 100644
--- a/stack/loadtests/README.md
+++ b/stack/loadtests/README.md
@@ -30,6 +30,9 @@ Deletes entities in order via name (prefix + entity number).
 ###runGetEntitiesByNameSequential.sh
 Retrieves entities one by one via name (prefix + entity number).
 
+###runGetEntitiesByUuidTest.sh
+Retrieves entities via UUID from a CSV file. Control the CSV feed pattern (circular=sequential,
random) using the csvFeedPattern configuration
+
 ###runLoadEntities.sh
 Creates entities in order via name (prefix + entity number).
 
@@ -42,15 +45,15 @@ Creates entities in order without names.
 ###runLoadSortableEntities.sh
 Creates sortable entities in order via name (prefix + entity number).
 
+###runOrgAppSetup.sh
+Runs the organization and app setup without doing a test.
+
 ###runRandomEntityByNameQueryTest.sh
 Retrieves random entities via name (prefix + entity number) using queries.
 
 ###runRandomEntityByNameTest.sh
 Retrieves random entities via name (prefix + entity number).
 
-###runRandomEntityByUuidTest.sh
-Retrieves random entities via UUID from a CSV file.
-
 ###runUpdateEntities.sh
 Updates entities in order via name (prefix + entity number).
 
@@ -74,7 +77,7 @@ Defaults listed are those that are specified by the Usergrid Gatling code,
not n
 * createOrg (**false**) - create the organization specified by the org configuration item
(will continue if the org already exists)
 * createApp (**false**) - create the application specified by the app configuration item
(will continue if the app already exists)
 * loadEntities (**false**) - load entities as part of setup, instead of as part of the test
-* scenarioType (**"nameRandomInfinite"**, "uuidRandomInfinite", "getByNameSequential", "getAllByCursor",
"loadEntities", "updateEntities", "deleteEntities", "auditGetCollectionEntities", "auditVerifyCollectionEntities")
- type of scenario to run, more details in test scripts section below
+* scenarioType (**"nameRandomInfinite"**, "uuidInfinite", "getByNameSequential", "getAllByCursor",
"loadEntities", "updateEntities", "deleteEntities", "auditGetCollectionEntities", "auditVerifyCollectionEntities")
- type of scenario to run, more details in test scripts section
 * rampUsers (**0**) - number of users to inject during the ramp up time
 * rampTime (**0**) - duration in seconds of the ramp up time
 * constantUsersPerSec (**0**) - number of users per second to inject during the constant
injection time (decimal ok)
@@ -110,6 +113,8 @@ Defaults listed are those that are specified by the Usergrid Gatling code,
not n
 * injectionList (**"rampUsers(10,60)"**) - custom injection pattern for CustomInjectionSimulation
(discussed below)
 * printFailedRequests (**true**) - prints the request and response on the console for failed
requests (those that fail more than *retryCount* times)
 * getViaQuery (**false**) - retrieve entities via query instead of via name or uuid
+* queryParams (**""**) - additional query parameters (currently used for get by entity or
by name)
+* csvFeedPattern (**"random"**) - pattern to use when feeding from a CSV ("random" is random,
"circular" goes through CSV sequentially and restarts from beginning when it reaches the end)
 
 The following settings are currently not used (were used by deprecated tests, but may be
valid in the future):
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/61bc825f/stack/loadtests/runGetEntitiesByUuid.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runGetEntitiesByUuid.sh b/stack/loadtests/runGetEntitiesByUuid.sh
new file mode 100755
index 0000000..545ab0d
--- /dev/null
+++ b/stack/loadtests/runGetEntitiesByUuid.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#               http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#LOAD_ENTITIES=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES=
+#SKIP_SETUP=
+#COLLECTION=
+#RETRY_COUNT=
+#END_CONDITION_TYPE=
+#END_MINUTES=
+#END_REQUEST_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
+
+#CSV_FEED_PATTERN=circular
+CSV_FEED_PATTERN=random
+
+die() { echo "$@" 1>&2 ; exit 1; }
+
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided.  Example is $0 RAMP_USERS
RAMP_TIME(seconds) UUID_FILENAME [[CSV_FEED_PATTERN] QUERY_PARAMS]"
+
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+UUID_FILENAME="$3"
+[ "$#" -ge 4 ] && CSV_FEED_PATTERN="$4"
+[ "$#" -ge 5 ] && QUERY_PARAMS="$5"
+
+shift $#
+
+SCENARIO_TYPE=uuidRandomInfinite
+
+#Compile everything
+mvn compile
+
+#Execute the test
+mvn gatling:execute \
+-DbaseUrl=${URL} \
+-DadminUser=${ADMIN_USER}  \
+-DadminPassword=${ADMIN_PASSWORD}  \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
+-DloadEntities=${LOAD_ENTITIES} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
+-DnumEntities=${NUM_ENTITIES} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
+-DretryCount=${RETRY_COUNT} \
+-DendConditionType=${END_CONDITION_TYPE} \
+-DendMinutes=${END_MINUTES} \
+-DendRequestCount=${END_REQUEST_COUNT} \
+-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC}    \
+-DconstantUsersDuration=${CONSTANT_USERS_DURATION}    \
+-DscenarioType=${SCENARIO_TYPE} \
+-DrampUsers=${RAMP_USERS}  \
+-DrampTime=${RAMP_TIME}  \
+-DuuidFilename=${UUID_FILENAME} \
+-DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
+-DqueryParams=${QUERY_PARAMS} \
+-DcsvFeedPattern=${CSV_FEED_PATTERN} \
+-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
+

http://git-wip-us.apache.org/repos/asf/usergrid/blob/61bc825f/stack/loadtests/runRandomEntityByUuidTest.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runRandomEntityByUuidTest.sh b/stack/loadtests/runRandomEntityByUuidTest.sh
deleted file mode 100755
index 01f4fe5..0000000
--- a/stack/loadtests/runRandomEntityByUuidTest.sh
+++ /dev/null
@@ -1,87 +0,0 @@
-#!/bin/bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#               http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-DIR="${BASH_SOURCE%/*}"
-if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
-. "$DIR/testConfig.sh"
-
-# from testConfig.sh
-#URL=
-#ADMIN_USER=
-#ADMIN_PASSWORD=
-#ORG=
-#APP=
-#AUTH_TYPE=
-#TOKEN_TYPE=
-#CREATE_ORG=
-#CREATE_APP=
-#LOAD_ENTITIES=
-#SANDBOX_COLLECTION=
-#NUM_ENTITIES=
-#SKIP_SETUP=
-#COLLECTION=
-#RETRY_COUNT=
-#END_CONDITION_TYPE=
-#END_MINUTES=
-#END_REQUEST_COUNT=
-#CONSTANT_USERS_PER_SEC=
-#CONSTANT_USERS_DURATION=
-
-die() { echo "$@" 1>&2 ; exit 1; }
-
-[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided.  Example is $0 RAMP_USERS
RAMP_TIME(seconds) UUID_FILENAME [QUERY_PARAMS]"
-
-RAMP_USERS="$1"
-RAMP_TIME="$2"
-UUID_FILENAME="$3"
-[ "$#" -ge 4 ] && QUERY_PARAMS="$4"
-
-shift $#
-
-SCENARIO_TYPE=uuidRandomInfinite
-
-#Compile everything
-mvn compile
-
-#Execute the test
-mvn gatling:execute \
--DbaseUrl=${URL} \
--DadminUser=${ADMIN_USER}  \
--DadminPassword=${ADMIN_PASSWORD}  \
--Dorg=${ORG} \
--Dapp=${APP} \
--DauthType=${AUTH_TYPE} \
--DtokenType=${TOKEN_TYPE} \
--DcreateOrg=${CREATE_ORG} \
--DcreateApp=${CREATE_APP} \
--DloadEntities=${LOAD_ENTITIES} \
--DsandboxCollection=${SANDBOX_COLLECTION} \
--DnumEntities=${NUM_ENTITIES} \
--DskipSetup=${SKIP_SETUP} \
--Dcollection=${COLLECTION} \
--DretryCount=${RETRY_COUNT} \
--DendConditionType=${END_CONDITION_TYPE} \
--DendMinutes=${END_MINUTES} \
--DendRequestCount=${END_REQUEST_COUNT} \
--DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC}    \
--DconstantUsersDuration=${CONSTANT_USERS_DURATION}    \
--DscenarioType=${SCENARIO_TYPE} \
--DrampUsers=${RAMP_USERS}  \
--DrampTime=${RAMP_TIME}  \
--DuuidFilename=${UUID_FILENAME} \
--DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
--DqueryParams=${QUERY_PARAMS} \
--Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
-

http://git-wip-us.apache.org/repos/asf/usergrid/blob/61bc825f/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
index 6647549..df0b325 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
@@ -80,6 +80,7 @@ object ConfigProperties {
   val MultiPropertySizeInK = "multiPropertySizeInK"
   val EntityNumberProperty = "entityNumberProperty"
   val QueryParams = "queryParams"
+  val CsvFeedPattern = "csvFeedPattern"
 
   val Values = Seq(Org,App,AdminUser,AdminPassword,BaseUrl,AuthType,TokenType,SkipSetup,CreateOrg,CreateApp,LoadEntities,
     ScenarioType,RampUsers,ConstantUsersPerSec,ConstantUsersDuration,UserSeed,AppUser,AppUserPassword,NumEntities,
@@ -88,7 +89,7 @@ object ConfigProperties {
     OrgCreationName,OrgCreationEmail,OrgCreationPassword,UpdateProperty,UpdateValue,EntityWorkerCount,EntityWorkerNum,
     UuidFilename,AuditUuidFilename,FailedUuidFilename,SandboxCollection,PurgeUsers,RetryCount,LaterThanTimestamp,
     EntityProgressCount,InjectionList,PrintFailedRequests,GetViaQuery,MultiPropertyPrefix,MultiPropertyCount,
-    MultiPropertySizeInK,EntityNumberProperty,QueryParams)
+    MultiPropertySizeInK,EntityNumberProperty,QueryParams,CsvFeedPattern)
 
   def isValid(str: String): Boolean = {
     Values.contains(str)
@@ -156,6 +157,7 @@ object ConfigProperties {
         case MultiPropertySizeInK => 1
         case EntityNumberProperty => ""
         case QueryParams => ""
+        case CsvFeedPattern => org.apache.usergrid.enums.CsvFeedPatternType.Random
       }
     } else {
       null

http://git-wip-us.apache.org/repos/asf/usergrid/blob/61bc825f/stack/loadtests/src/main/scala/org/apache/usergrid/enums/CsvFeedPatternType.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/CsvFeedPatternType.scala
b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/CsvFeedPatternType.scala
new file mode 100644
index 0000000..6c1ab8d
--- /dev/null
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/CsvFeedPatternType.scala
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.enums
+
+/**
+ * Created by mdunker on 8/31/15.
+ */
+object CsvFeedPatternType {
+  val Random = "random"
+  val Circular = "circular"
+
+  val Values = Seq(Random,Circular)
+  def isValid(str: String): Boolean = {
+    Values.contains(str)
+  }
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/61bc825f/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index 38a70ff..0ab2cd6 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -20,7 +20,7 @@ import io.gatling.core.Predef._
 import io.gatling.core.feeder.RecordSeqFeederBuilder
 import io.gatling.http.Predef._
 import org.apache.usergrid.datagenerators.FeederGenerator
-import org.apache.usergrid.enums.{EndConditionType, AuthType}
+import org.apache.usergrid.enums.{CsvFeedPatternType, EndConditionType, AuthType}
 import org.apache.usergrid.helpers.Extractors._
 import org.apache.usergrid.helpers.{Headers, Utils}
 import org.apache.usergrid.settings.Settings
@@ -58,8 +58,11 @@ object EntityCollectionScenarios {
   }
 
   def uuidFeeder(): RecordSeqFeederBuilder[String] = {
-
-    csv(Settings.feedUuidFilename).random
+    if (Settings.csvFeedPattern == CsvFeedPatternType.Circular) {
+      csv(Settings.feedUuidFilename).circular
+    } else {
+      csv(Settings.feedUuidFilename).random
+    }
   }
 
   /*
@@ -160,7 +163,7 @@ object EntityCollectionScenarios {
       .check(status.is(200))
   )
 
-  val getRandomEntitiesByUuid = scenario("Get entities by uuid randomly")
+  val getRandomEntitiesByUuid = scenario("Get entities by uuid")
     .exec(injectTokenIntoSession())
     .exec(injectAuthType())
     .doIfOrElse(_ => Settings.endConditionType == EndConditionType.MinutesElapsed) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/61bc825f/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index 0a800cc..e9dd2f2 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -180,6 +180,7 @@ object Settings {
   val getViaQuery:Boolean = initBoolSetting(ConfigProperties.GetViaQuery)
   private val queryParamConfig = initStrSetting(ConfigProperties.QueryParams)
   val queryParamMap: Map[String,String] = mapFromQueryParamConfigString(queryParamConfig)
+  val csvFeedPattern = initStrSetting(ConfigProperties.CsvFeedPattern)
 
   val multiPropertyPrefix = initStrSetting(ConfigProperties.MultiPropertyPrefix)
   val multiPropertyCount:Int = initIntSetting(ConfigProperties.MultiPropertyCount)
@@ -454,6 +455,7 @@ object Settings {
     } else {
       println(s"SearchLimit:$searchLimit  SearchQuery:$searchQuery")
     }
+    if (queryParamConfig != "") println(s"Extra query params: $queryParamConfig")
     println()
     println(s"Overall: NumEntities:$totalNumEntities  Seed:$overallEntitySeed  Workers:$entityWorkerCount")
     println(s"Worker:  NumEntities:$numEntities  Seed:$entitySeed  WorkerNum:$entityWorkerNum")
@@ -462,7 +464,10 @@ object Settings {
     println(s"Constant: UsersPerSec:$constantUsersPerSec  Time:$constantUsersDuration")
     println(s"EndCondition:$endConditionStr")
     println()
-    if (feedUuids) println(s"Feed CSV: $feedUuidFilename")
+    if (feedUuids) {
+      println(s"Feed CSV: $feedUuidFilename")
+      println(s"Feed pattern: $csvFeedPattern")
+    }
     if (feedAuditUuids) println(s"Audit Feed CSV: $feedAuditUuidFilename")
     if (captureUuids) println(s"Capture CSV:$captureUuidFilename")
     if (captureAuditUuids) {


Mime
View raw message