usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sfeld...@apache.org
Subject [34/50] incubator-usergrid git commit: Fixes worker issue in cloud formation
Date Mon, 16 Mar 2015 23:38:02 GMT
Fixes worker issue in cloud formation


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c319d07d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c319d07d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c319d07d

Branch: refs/heads/two-dot-o
Commit: c319d07d8c09d2e5988f58faad1dd42cff85cbe8
Parents: 9aec790
Author: Todd Nine <tnine@apigee.com>
Authored: Wed Mar 11 19:33:54 2015 -0600
Committer: Todd Nine <tnine@apigee.com>
Committed: Wed Mar 11 19:33:54 2015 -0600

----------------------------------------------------------------------
 .../src/main/groovy/configure_usergrid.groovy   |   8 +-
 stack/awscluster/ugcluster-cf.json              | 150 ++-----------------
 2 files changed, 23 insertions(+), 135 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c319d07d/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 8af5884..fb98368 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -51,6 +51,9 @@ def esShards = numEsNodes*2;
 def esReplicas = 1;
 
 def tomcatThreads = System.getenv().get("TOMCAT_THREADS")
+
+def workerCount = System.getenv().get("INDEX_WORKER_COUNT")
+
 //temporarily set to equal since we now have a sane tomcat thread calculation
 def hystrixThreads = tomcatThreads
 
@@ -58,7 +61,6 @@ def hystrixThreads = tomcatThreads
 def ec2Region = System.getenv().get("EC2_REGION")
 def cassEc2Region = ec2Region.replace("-1", "")
 
-
 NodeRegistry registry = new NodeRegistry();
 
 def selectResult = registry.searchNode('cassandra')
@@ -195,6 +197,10 @@ usergrid.queue.region=${ec2Region}
 usergrid.scheduler.enabled=true
 usergrid.scheduler.job.workers=1
 
+
+#Set our ingest rate
+elasticsearch.worker_count=${workerCount}
+
 """
 
 println usergridConfig

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c319d07d/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index 0c2a9d2..e9db671 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -46,6 +46,12 @@
             ],
             "ConstraintDescription": "must be valid instance type."
         },
+      "RestIndexWorkers":{
+        "Description": "The number of index workers to ingest ElasticSearch batch operations
per tomcat",
+        "Type": "Number",
+        "Default": "8",
+        "MinValue": "3"
+      },
       "TomcatThreadsPerCore": {
         "Description": "Number of threads to configure tomcat for per core",
         "Type": "Number",
@@ -289,7 +295,7 @@
                 }
             }
         },
-       "GraphiteAutoScalingLaunchConfiguration":{
+        "GraphiteAutoScalingLaunchConfiguration":{
          "Type":"AWS::AutoScaling::LaunchConfiguration",
          "Properties":{
             "UserData":{
@@ -464,7 +470,7 @@
                 }
             }
         },
-            "OpsCenterUser": {
+        "OpsCenterUser": {
                         "Type": "AWS::IAM::User",
                         "Properties": {
                             "Path": "/",
@@ -485,7 +491,7 @@
                             ]
                         }
                     },
-                    "OpsCenterKey": {
+        "OpsCenterKey": {
                         "Type": "AWS::IAM::AccessKey",
                         "Properties": {
                             "UserName": {
@@ -493,7 +499,7 @@
                             }
                         }
                     },
-                   "OpsCenterAutoScalingLaunchConfiguration":{
+        "OpsCenterAutoScalingLaunchConfiguration":{
                      "Type":"AWS::AutoScaling::LaunchConfiguration",
                      "Properties":{
                         "UserData":{
@@ -619,7 +625,7 @@
 
                      }
                   },
-                    "OpsCenterAutoScalingGroup": {
+        "OpsCenterAutoScalingGroup": {
                         "Type": "AWS::AutoScaling::AutoScalingGroup",
                         "Version": "2014-07-24",
                         "Properties": {
@@ -767,7 +773,7 @@
                 }
             }
         },
-      "CassAutoScalingLaunchConfiguration":{
+        "CassAutoScalingLaunchConfiguration":{
          "Type":"AWS::AutoScaling::LaunchConfiguration",
          "Properties":{
             "UserData":{
@@ -955,7 +961,7 @@
                 }
             }
         },
-      "ESMasterAutoScalingLaunchConfiguration":{
+        "ESMasterAutoScalingLaunchConfiguration":{
          "Type":"AWS::AutoScaling::LaunchConfiguration",
          "Properties":{
             "UserData":{
@@ -1137,131 +1143,6 @@
                 }
             }
         },
-      "RestAutoScalingLaunchConfiguration":{
-         "Type":"AWS::AutoScaling::LaunchConfiguration",
-         "Properties":{
-            "UserData":{
-               "Fn::Base64":{
-                  "Fn::Join":[
-                     "",
-                     [
-                        "#!/bin/bash -ex\n",
-                        "# REST SERVER STARTUP \n",
-                        "exec >/var/log/usergrid-bootstrap.log 2>&1\n",
-                        "\n",
-                        "mkdir -p /usr/share/usergrid\n",
-                        "\n",
-                        "# create script that sets our environment variables\n",
-                        "cat >/etc/profile.d/usergrid-env.sh <<EOF\n",
-                        "alias sudo='sudo -E'\n",
-                        "\n",
-                        "export TYPE=rest\n",
-                        "export STACK_NAME=", { "Ref":"AWS::StackName" }, "\n",
-                        "export YOURKIT=", { "Ref":"InstallYourkit" }, "\n",
-                        "export DNS_NAME=", { "Ref":"DnsSubDomain" }, "\n",
-                        "export DNS_DOMAIN=", { "Ref":"DnsDomain" }, "\n",
-                        "export PUBLIC_HOSTNAME=`(curl -s http://169.254.169.254/latest/meta-data/public-hostname)`\n",
-                        "export INTERNAL_HOSTNAME=`(curl http://169.254.169.254/latest/meta-data/local-ipv4)`\n",
-                        "export ELB_NAME=", { "Ref":"RestElasticLoadBalancer" }, "\n",
-                        "\n",
-                        "export EC2_INSTANCE_ID=`ec2metadata --instance-id`\n",
-                        "export EC2_REGION=", { "Ref":"AWS::Region" }, "\n",
-                        "export EC2_URL=https://ec2.amazonaws.com/\n", "\n",
-                        "export REST_SECURITY_GROUP_NAME=", { "Ref":"RestSecurityGroup" },
"\n",
-                        "export DB_SECURITY_GROUP_NAME=", { "Ref":"CassSecurityGroup" },
"\n",
-                        "\n",
-                        "export CASSANDRA_CLUSTER_NAME=", { "Ref":"CassClusterName" }, "\n",
-                        "export CASSANDRA_KEYSPACE_NAME=usergrid", "\n",
-                        "export CASSANDRA_NUM_SERVERS=", { "Ref":"CassNumServers" }, "\n",
-                        "export GRAPHITE_NUM_SERVERS=", { "Ref":"GraphiteNumServers" }, "\n",
-                        "export TOMCAT_NUM_SERVERS=", { "Ref":"RestMinServers" }, "\n",
-                        "\n",
-                        "export CASSANDRA_REPLICATION_FACTOR=", { "Ref":"CassReplicationFactor"
}, "\n",
-                        "\n",
-                        "export CASSANDRA_READ_CONSISTENCY=", { "Ref":"CassReadConsistency"
}, "\n",
-                        "\n",
-                        "export CASSANDRA_WRITE_CONSISTENCY=", { "Ref":"CassWriteConsistency"
}, "\n",
-                        "\n",
-
-                        "export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
-                        "export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",
-                         "\n",
-                        "export RELEASE_BUCKET=", { "Ref":"ReleaseBucket" }, "\n",
-                        "\n",
-                        "export NUM_THREAD_PROC=", { "Ref":"TomcatThreadsPerCore" }, "\n",
-                        "\n",
-                        "export SUPER_USER_EMAIL=", { "Ref":"SuperUserEmail" }, "\n",
-                        "export TEST_ADMIN_USER_EMAIL=", { "Ref":"TestAdminUserEmail" },
"\n",
-                        "\n",
-                        "EOF\n",
-                        "\n",
-                        "# put AWS creds in environment\n",
-                        "cat >/etc/profile.d/aws-credentials.sh <<EOF\n",
-                        "export AWS_ACCESS_KEY=", { "Ref":"RestKey" }, "\n",
-                        "export AWS_SECRET_KEY=", { "Fn::GetAtt":[ "RestKey", "SecretAccessKey"
] }, "\n",
-                        "EOF\n",
-                        "\n",
-                        "# setup s3cmd (will be installed by init script) \n",
-                        "cat >/etc/s3cfg <<EOF\n",
-                        "access_key=", { "Ref":"RestKey" }, "\n",
-                        "secret_key=", { "Fn::GetAtt":[ "RestKey", "SecretAccessKey" ] },
"\n",
-                        "EOF\n",
-                        "chmod 644 /etc/s3cfg\n",
-                        "ln -s /etc/s3cfg ~ubuntu/.s3cfg\n",
-                        "ln -s /etc/s3cfg ~root/.s3cfg\n",
-                        "\n",
-                        "# download usergrid and init script bundle from S3\n",
-                        "wget -O- -q http://s3tools.org/repo/deb-all/stable/s3tools.key |
apt-key add -\n",
-                        "wget -O/etc/apt/sources.list.d/s3tools.list http://s3tools.org/repo/deb-all/stable/s3tools.list\n",
-                        "apt-get update\n",
-                        "apt-get -y install s3cmd\n",
-                        "cd /usr/share/usergrid\n",
-                        "s3cmd --config=/etc/s3cfg get s3://", {"Ref": "ReleaseBucket"},
"/awscluster-1.0-SNAPSHOT-any.tar.gz\n",
-                        "s3cmd --config=/etc/s3cfg get s3://", {"Ref": "ReleaseBucket"},
"/ROOT.war\n",
-                        "tar xvf awscluster-1.0-SNAPSHOT-any.tar.gz\n",
-                        "rm -fr awscluster-1.0-SNAPSHOT-any.tar.gz\n",
-                        "mv ROOT.war webapps/ROOT.war\n",
-                        "chmod 755 ./init_instance/*.sh\n",
-                        "cd ./init_instance\n",
-                        "# Init as a REST intance \n",
-                        "sh ./init_rest_server.sh\n"
-                     ]
-                  ]
-               }
-            },
-            "KeyName":{
-               "Ref":"KeyPair"
-            },
-            "ImageId":{
-               "Fn::FindInMap":[
-                  "AWSRegionArch2AMI",
-                  {
-                     "Ref":"AWS::Region"
-                  },
-                  {
-                     "Fn::FindInMap":[
-                        "AWSInstanceType2Arch",
-                        {
-                           "Ref":"RestInstanceType"
-                        },
-                        "Arch"
-                     ]
-                  }
-               ]
-            },
-            "InstanceType":{
-               "Ref":"RestInstanceType"
-            },
-            "IamInstanceProfile":{
-               "Ref":"RootInstanceProfile"
-            },
-            "SecurityGroups":[
-               {
-                  "Ref":"RestSecurityGroup"
-               }
-            ]
-         }
-      },
         "ESAutoScalingLaunchConfiguration":{
                  "Type":"AWS::AutoScaling::LaunchConfiguration",
                  "Properties":{
@@ -1378,7 +1259,7 @@
                             ]
                     }
               },
-                "ESAutoScalingGroup": {
+        "ESAutoScalingGroup": {
                     "Type": "AWS::AutoScaling::AutoScalingGroup",
                     "Version": "2009-05-15",
                     "Properties": {
@@ -1442,7 +1323,7 @@
                         }
                     }
                 },
-              "RestAutoScalingLaunchConfiguration":{
+        "RestAutoScalingLaunchConfiguration":{
                  "Type":"AWS::AutoScaling::LaunchConfiguration",
                  "Properties":{
                     "UserData":{
@@ -1487,6 +1368,7 @@
                                 "\n",
                                 "export CASSANDRA_WRITE_CONSISTENCY=", { "Ref":"CassWriteConsistency"
}, "\n",
                                 "\n",
+                               "export INDEX_WORKER_COUNT=", { "Ref":"RestIndexWorkers" },
"\n",
 
                                 "export ES_CLUSTER_NAME=", { "Ref":"ESClusterName" }, "\n",
                                 "export ES_NUM_SERVERS=", { "Ref":"ESNumServers" }, "\n",


Mime
View raw message