usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [4/4] git commit: Fixes to get 100% Core module tests working again, and they now run in 11m on my 16gb Macbook Pro
Date Tue, 22 Jul 2014 17:10:15 GMT
Fixes to get 100% Core module tests working again, and they now run in 11m on my 16gb Macbook
Pro


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/54f045fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/54f045fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/54f045fc

Branch: refs/heads/two-dot-o
Commit: 54f045fc670f4f6572e9bccefde07ae4a4aa3fd2
Parents: 173ebc8
Author: Dave Johnson <dmjohnson@apigee.com>
Authored: Tue Jul 22 12:58:45 2014 -0400
Committer: Dave Johnson <dmjohnson@apigee.com>
Committed: Tue Jul 22 12:58:45 2014 -0400

----------------------------------------------------------------------
 .../main/resources/usergrid-default.properties  | 78 +++++++++++-------
 stack/core/pom.xml                              | 86 +++++++++++++++-----
 .../usergrid/persistence/PathQueryIT.java       |  2 +-
 .../src/test/resources/dynamic-test.properties  | 21 -----
 .../src/test/resources/usergrid-CHOP.properties | 17 ++++
 .../shard/count/NodeShardApproximationImpl.java |  9 +-
 .../cassandra/ManagementServiceImpl.java        | 39 ++++-----
 .../services/AbstractCollectionService.java     |  2 +-
 8 files changed, 153 insertions(+), 101 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54f045fc/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index dd22d93..07483f4 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -27,23 +27,27 @@
 # EM for old Entity Manager, CP for new Core Persistence
 usergrid.persistence=CP
 
-## Corepersistence properties
+
+###############################################################################
+#
+# Corepersistence properties
+
+#cassandra.hosts=127.0.0.1  # will be set by CpSetup
+#cassandra.port=9160        # will be set by CpSetup
 cassandra.embedded=true
-cassandra.hosts=127.0.0.1
-cassandra.port=9160
 cassandra.version=1.2
 cassandra.cluster_name=Usergrid
 cassandra.timeout=2000
 
-# Should be set to a larger value than Hystrix thread pool size
-#cassandra.connections=800
-
 collections.keyspace=Usergrid_Applications
 collections.keyspace.strategy.options=replication_factor:1
 collections.keyspace.strategy.class=org.apache.cassandra.locator.SimpleStrategy
 
 collection.stage.transient.timeout=60
 
+hystrix.threadpool.graph_user.coreSize=40
+hystrix.threadpool.graph_async.coreSize=40
+
 elasticsearch.embedded=true
 elasticsearch.cluster_name=usergrid_test
 elasticsearch.index_prefix=usergrid
@@ -52,14 +56,16 @@ elasticsearch.port=9300
 
 index.query.limit.default=100
 
-# Whether to user the remote Cassandra cluster or not
+# Max Cassandra connections, applies to both CP and EM
+cassandra.connections=600
 
-# URL for local testing Cassandra cluster
-cassandra.url=localhost:9160
 
-#The number of thrift connections to open per cassandra node.
-cassandra.connections=50
+###############################################################################
+#
+# Classic Entity Manager properties
 
+# URL for local testing Cassandra cluster
+cassandra.url=localhost:9160
 
 # Name of Cassandra cluster
 cassandra.cluster=Test Cluster
@@ -86,26 +92,49 @@ cassandra.writecl=QUORUM
 #Even numbers for RF can potentially case the locks to fail, via "split brain" when read
at QUORUM on lock verification
 cassandra.lock.keyspace=Locks
 
+
+###############################################################################
+#
+# General properties
+
 # false to disable test features
 usergrid.test=false
 
-#Properties to control the number of buckets in the index.
+# Properties to control the number of buckets in the index.
 usergrid.index.defaultbucketsize=20
 usergrid.counter.skipAggregate=false
 usergrid.version.database=1.0.0
 usergrid.version.schema=1.0.0
 usergrid.version.properties=1.0.0
+
 # build number for display
 usergrid.version.build=${version}
 
 usergird.service.packages=com.usergrid.services;org.apache.usergrid.services;baas.io
 
-#Batch submit counters ever 1000 updates
+# Batch submit counters ever 1000 updates
 usergrid.counter.batch.size=1000
 
-#Submit batcher every 30 seconds
+# Submit batcher every 30 seconds
 usergrid.counter.batch.interval=30
 
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_KEY=
+usergrid.binary.bucketname=usergrid-test
+
+usergrid.test.sample_data_url=
+
+# Disable Mongo API Server
+usergrid.mongo.disable=true
+
+# Disable WebSocket Server
+usergrid.websocket.disable=true
+
+
+###############################################################################
+#
+# Authentication properties
+
 #usergrid.auth.token_secret_salt=super secret token value
 #usergrid.auth.token_expires_from_last_use=false
 #usergrid.auth.token_refresh_reuses_id=false
@@ -141,9 +170,10 @@ usergrid.temp.files=/tmp/usergrid
 #The timeout in locks from reading messages transactionally from a queue.  Number of seconds
to wait
 usergrid.queue.lock.timeout=30
 
-######
-#Scheduler setup
-######
+
+###############################################################################
+#
+# Scheduler setup
 
 #Time in milliseconds that a job can be started without a heartbeat before being considered
dead.  
 #Note that this must be high enough so that jobs that are iteration based can run an iteration
and update the heartbeat
@@ -165,17 +195,10 @@ zookeeper.maxAttempts = 3
 
 swagger.basepath=http://localhost:8080
 
-AWS_ACCESS_KEY_ID=
-AWS_SECRET_KEY=
-usergrid.binary.bucketname=usergrid-test
-
-usergrid.test.sample_data_url=
-
-# Disable Mongo API Server
-usergrid.mongo.disable=true
 
-# Disable WebSocket Server
-usergrid.websocket.disable=true
+###############################################################################
+#
+# Mail and activation setup
 
 mail.transport.protocol=
 mail.smtps.host=
@@ -246,7 +269,6 @@ usergrid.management.email.sysadmin-admin-activated=\
 ###############################################################################
 #
 # Organization approval flow
-#
 
 # email to sysadmin to approve and activate new organization
 usergrid.management.email.sysadmin-organization-activation=\

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54f045fc/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index 91ca8bb..6cd5c49 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -27,7 +27,7 @@
   <properties>
     <!-- Up to 30x parallelism can be used -->
     <core.it.forkCount>2</core.it.forkCount>
-    <core.it.suite.concurrency>5</core.it.suite.concurrency>
+    <core.it.suite.concurrency>2</core.it.suite.concurrency>
   </properties>
 
   <artifactId>usergrid-core</artifactId>
@@ -130,27 +130,31 @@
   </profiles>
 
   <build>
-    <resources>
-      <resource>
-        <directory>src/main/resources</directory>
-        <filtering>true</filtering>
-        <includes>
-          <include>**/*.xml</include>
-        </includes>
-      </resource>
-    </resources>
-
-    <testResources>
-      <testResource>
-        <directory>src/test/resources</directory>
-        <filtering>true</filtering>
-        <includes>
-          <include>**/*.yaml</include>
-          <include>**/*.xml</include>
-          <include>**/*.properties</include>
-        </includes>
-      </testResource>
-    </testResources>
+
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <filtering>true</filtering>
+                <includes>
+                    <include>**/*.sh</include>
+                    <include>**/stack.json</include>
+                    <include>**/*.properties</include>
+                    <include>**/*.xml</include> 
+                </includes>
+            </resource>
+        </resources>
+
+        <testResources> 
+            <testResource> 
+                <directory>src/test/resources</directory> 
+                <filtering>true</filtering> 
+                <includes>
+                    <include>**/*.yaml</include>
+                    <include>**/*.properties</include> 
+                    <include>**/*.xml</include> 
+                </includes> 
+            </testResource> 
+        </testResources>
 
     <plugins>
 
@@ -209,6 +213,33 @@
           <target>1.7</target>
         </configuration>
       </plugin>
+
+<!--            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>2.4</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.usergrid.chop</groupId>
+                <artifactId>chop-maven-plugin</artifactId>
+                <version>2.0.0-SNAPSHOT</version>
+                <configuration>
+                    <username>${chop.coordinator.username}</username>
+                    <password>${chop.coordinator.password}</password>
+                    <endpoint>https://${chop.coordinator.url}:8443</endpoint>
+                    <testPackageBase>ctest</testPackageBase>
+                    <runnerCount>1</runnerCount>
+                </configuration>
+            </plugin>-->
+
     </plugins>
   </build>
 
@@ -505,6 +536,17 @@
         <scope>test</scope>
     </dependency>
 
+<!--    <dependency>
+        <groupId>org.apache.usergrid.chop</groupId>
+        <artifactId>chop-api</artifactId>
+        <version>2.0.0-SNAPSHOT</version>
+    </dependency>
+
+    <dependency>
+        <groupId>org.apache.usergrid.chop</groupId>
+        <artifactId>chop-stack</artifactId>
+        <version>2.0.0-SNAPSHOT</version>
+    </dependency>-->
 
   </dependencies>
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54f045fc/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java
index fb5e679..bdaef0a 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PathQueryIT.java
@@ -78,7 +78,7 @@ public class PathQueryIT extends AbstractCoreIT {
         // query the users, ignoring page boundaries
         Results results = em.searchCollection( em.getApplicationRef(), "users", userQuery
);
         PagingResultsIterator pri = new PagingResultsIterator( results );
-        long count = 2;
+        int count = 2;
         while ( pri.hasNext() ) {
             Entity e = ( Entity ) pri.next();
             assertEquals( count++, e.getProperty( "index" ) );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54f045fc/stack/core/src/test/resources/dynamic-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/dynamic-test.properties b/stack/core/src/test/resources/dynamic-test.properties
deleted file mode 100644
index d58bad6..0000000
--- a/stack/core/src/test/resources/dynamic-test.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-# The properties are not the actual configuration properties but
-# safe dynamic property defaults for our testing via IDE or Maven
-
-collections.keyspace=Usergrid_Applications
-collections.keyspace.strategy.options=replication_factor:1
-collections.keyspace.strategy.class=SimpleStrategy
-collection.stage.transient.timeout=60
-
-cassandra.connections=10
-cassandra.port=9160
-cassandra.version=1.2
-cassandra.hosts=localhost
-cassandra.cluster_name=Usergrid
-cassandra.timeout=5000
-
-index.query.limit.default=10
-
-elasticsearch.indexname=Usergrid
-elasticsearch.embedded=true
-elasticsearch.force-refresh=false
-

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54f045fc/stack/core/src/test/resources/usergrid-CHOP.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-CHOP.properties b/stack/core/src/test/resources/usergrid-CHOP.properties
new file mode 100644
index 0000000..a09c0fb
--- /dev/null
+++ b/stack/core/src/test/resources/usergrid-CHOP.properties
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Overrides

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54f045fc/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
index 7dc763f..6fef61b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationImpl.java
@@ -134,18 +134,19 @@ public class NodeShardApproximationImpl implements NodeShardApproximation
{
 
         writeLockLock.lock();
 
+        final MutationBatch batch;
+
         try {
             flushPending = currentCounter;
             currentCounter = new Counter();
+
+            //copy to the batch outside of the command for performance
+            batch = nodeShardCounterSerialization.flush( flushPending );
         }
         finally {
             writeLockLock.unlock();
         }
 
-
-        //copy to the batch outside of the command for performance
-        final MutationBatch batch = nodeShardCounterSerialization.flush( flushPending );
-
         /**
          * Execute the command in hystrix to avoid slamming cassandra
          */

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54f045fc/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 53ed9ed..b30c99e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -1825,16 +1825,21 @@ public class ManagementServiceImpl implements ManagementService {
     }
 
 
-    public String getSecret( UUID applicationId, AuthPrincipalType type, UUID entityId, String
entityType ) throws Exception {
-        if ( AuthPrincipalType.ORGANIZATION.equals( type ) || AuthPrincipalType.APPLICATION.equals(
type ) ) {
-            UUID ownerId =
-                    AuthPrincipalType.APPLICATION_USER.equals( type ) ? applicationId : smf.getManagementAppId();
+    public String getSecret( UUID applicationId, AuthPrincipalType type, UUID entityId )
throws Exception {
+
+        if ( AuthPrincipalType.ORGANIZATION.equals( type )) {  
+            UUID ownerId = smf.getManagementAppId();
+            return getCredentialsSecret( readUserToken( ownerId, entityId, Group.ENTITY_TYPE
) );
+
+        } else if ( AuthPrincipalType.APPLICATION.equals( type ) ) {
+            UUID ownerId = smf.getManagementAppId();
+            return getCredentialsSecret( readUserToken( ownerId, entityId, Application.ENTITY_TYPE
) );
 
-            return getCredentialsSecret( readUserToken( ownerId, entityId, entityType ) );
         }
         else if ( AuthPrincipalType.ADMIN_USER.equals( type ) || AuthPrincipalType.APPLICATION_USER.equals(
type ) ) {
-            return getCredentialsSecret( readUserPasswordCredentials( applicationId, entityId,
entityType ) );
+            return getCredentialsSecret( readUserPasswordCredentials( applicationId, entityId,
User.ENTITY_TYPE ) );
         }
+
         throw new IllegalArgumentException( "Must specify an admin user, organization or
application principal" );
     }
 
@@ -1847,7 +1852,7 @@ public class ManagementServiceImpl implements ManagementService {
 
     @Override
     public String getClientSecretForOrganization( UUID organizationId ) throws Exception
{
-        return getSecret( smf.getManagementAppId(), AuthPrincipalType.ORGANIZATION, organizationId,
Group.ENTITY_TYPE );
+        return getSecret( smf.getManagementAppId(), AuthPrincipalType.ORGANIZATION, organizationId
);
     }
 
 
@@ -1859,7 +1864,7 @@ public class ManagementServiceImpl implements ManagementService {
 
     @Override
     public String getClientSecretForApplication( UUID applicationId ) throws Exception {
-        return getSecret( smf.getManagementAppId(), AuthPrincipalType.APPLICATION, applicationId,
Application.ENTITY_TYPE );
+        return getSecret( smf.getManagementAppId(), AuthPrincipalType.APPLICATION, applicationId
);
     }
 
 
@@ -1900,14 +1905,7 @@ public class ManagementServiceImpl implements ManagementService {
         }
         AccessInfo access_info = null;
 
-        String entityType = null;
-        if ( type.equals( AuthPrincipalType.APPLICATION )) {
-            entityType = Application.ENTITY_TYPE;
-        } else {
-            entityType = Group.ENTITY_TYPE;
-        } 
-
-        if ( clientSecret.equals( getSecret( smf.getManagementAppId(), type, uuid, entityType
) ) ) {
+        if ( clientSecret.equals( getSecret( smf.getManagementAppId(), type, uuid ) ) ) {
 
             String token = getTokenForPrincipal( ACCESS, null, smf.getManagementAppId(),
type, uuid, ttl );
 
@@ -1944,15 +1942,8 @@ public class ManagementServiceImpl implements ManagementService {
             return null;
         }
 
-        String entityType = null;
-        if ( type.equals( AuthPrincipalType.APPLICATION )) {
-            entityType = Application.ENTITY_TYPE;
-        } else {
-            entityType = Group.ENTITY_TYPE;
-        } 
-
         PrincipalCredentialsToken token = null;
-        if ( clientSecret.equals( getSecret( smf.getManagementAppId(), type, uuid, entityType
))) {
+        if ( clientSecret.equals( getSecret( smf.getManagementAppId(), type, uuid))) {
             if ( type.equals( AuthPrincipalType.APPLICATION ) ) {
                 ApplicationInfo app = getApplicationInfo( uuid );
                 token = new PrincipalCredentialsToken( new ApplicationPrincipal( app ),

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54f045fc/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
index 026cc7b..0b1e603 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
@@ -271,7 +271,7 @@ public class AbstractCollectionService extends AbstractService {
 
         checkPermissionsForEntity( context, id );
 
-        Entity item = em.get( new SimpleEntityRef( getEntityType(), id) );
+        Entity item = em.get( id );
 
         if ( item != null ) {
             validateEntityType( item, id );


Mime
View raw message