usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mdun...@apache.org
Subject usergrid git commit: 1. Add missing start for delete queue workers. 2. Add dropping of application local keyspace for testing. 3. Testing changes.
Date Tue, 22 Aug 2017 00:02:15 GMT
Repository: usergrid
Updated Branches:
  refs/heads/collectionClearJob c201f1f67 -> 3f7afcd88


1. Add missing start for delete queue workers.
2. Add dropping of application local keyspace for testing.
3. Testing changes.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3f7afcd8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3f7afcd8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3f7afcd8

Branch: refs/heads/collectionClearJob
Commit: 3f7afcd8836ed9b589b69523d8bcc507a739d912
Parents: c201f1f
Author: Mike Dunker <mdunker@google.com>
Authored: Mon Aug 21 17:01:48 2017 -0700
Committer: Mike Dunker <mdunker@google.com>
Committed: Mon Aug 21 17:01:48 2017 -0700

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        |  3 +--
 .../asyncevents/AsyncEventServiceImpl.java      |  4 +++
 .../index/CollectionSettingsImpl.java           |  1 -
 .../usergrid/persistence/EntityManager.java     |  2 +-
 .../persistence/entities/Application.java       | 26 ++++++++++++++++++++
 .../usergrid/persistence/CoreSchemaManager.java | 16 ++++++++----
 .../usergrid/persistence/RebuildIndexTest.java  | 24 +++++++++++++-----
 .../resources/usergrid-custom-test.properties   |  2 +-
 .../resources/usergrid-custom-test.properties   |  2 +-
 .../AbstractServiceNotificationIT.java          |  4 +--
 .../gcm/NotificationsServiceIT.java             |  2 +-
 .../resources/usergrid-custom-test.properties   |  2 +-
 12 files changed, 67 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index cdb4fc7..68c4ef0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2471,10 +2471,9 @@ public class CpEntityManager implements EntityManager {
 
         final Entity entity;
 
-        //this is the fall back, why isn't this writt
         if ( entityType == null ) {
              return null;
-//            throw new EntityNotFoundException( String.format( "Counld not find type for
uuid {}", uuid ) );
+//            throw new EntityNotFoundException( String.format( "Could not find type for
uuid {}", uuid ) );
         }
 
         entity = get( new SimpleEntityRef( entityType, uuid ) );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 79a80c0..428772f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -875,6 +875,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
             startWorker(AsyncEventQueueType.UTILITY);
         }
 
+        for (int i = 0; i < deleteCount; i++) {
+            startWorker(AsyncEventQueueType.DELETE);
+        }
+
         if( indexQueue instanceof SNSQueueManagerImpl) {
             logger.info("Queue manager implementation supports dead letters, start dead letter
queue workers.");
             for (int i = 0; i < indexDeadCount; i++) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
index 921777a..76f0b4b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
@@ -20,7 +20,6 @@ package org.apache.usergrid.corepersistence.index;
 
 import com.google.common.base.Optional;
 import com.google.inject.Inject;
-import com.google.inject.Singleton;
 import org.apache.usergrid.persistence.map.MapManager;
 import org.apache.usergrid.utils.JsonUtils;
 import org.slf4j.Logger;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index ae4623d..c9752c3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -743,7 +743,7 @@ public interface EntityManager {
 
     /**
      * Add a new index to the application for scale
-     * @param suffix unique indentifier for additional index
+     * @param newIndexName unique identifier for additional index
      * @param shards number of shards
      * @param replicas number of replicas
      * @param writeConsistency only "one, quorum, or all"

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
index 790b4d9..0a4360f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
@@ -48,6 +48,32 @@ public class Application extends TypedEntity implements Serializable {
 
     public static final String COLLECTION_ACTIVITIES = "activities";
 
+    public static final String COLLECTION_EVENTS = "events";
+
+    public static final String COLLECTION_FOLDERS = "folders";
+
+    public static final String COLLECTION_DEVICES = "devices";
+
+    public static final String COLLECTION_NOTIFICATIONS = "notifications";
+
+    public static final String COLLECTION_ROLES = "roles";
+
+    public static boolean isCustomCollectionName(String collectionName) {
+        switch (collectionName.toLowerCase()) {
+            case COLLECTION_USERS:
+            case COLLECTION_GROUPS:
+            case COLLECTION_ASSETS:
+            case COLLECTION_ACTIVITIES:
+            case COLLECTION_EVENTS:
+            case COLLECTION_FOLDERS:
+            case COLLECTION_DEVICES:
+            case COLLECTION_NOTIFICATIONS:
+            case COLLECTION_ROLES:
+                return false;
+        }
+        return true;
+    }
+
     @EntityProperty(indexed = true, fulltextIndexed = false, required = true, mutable = false,
aliasProperty = true,
             basic = true)
     protected String name;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
index c064751..1d51c25 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.persistence;
 
 
 import org.apache.usergrid.locking.LockManager;
+import org.apache.usergrid.persistence.core.CassandraConfig;
 import org.apache.usergrid.persistence.core.CassandraFig;
 import org.apache.usergrid.persistence.core.datastax.CQLUtils;
 import org.apache.usergrid.persistence.core.datastax.DataStaxCluster;
@@ -37,14 +38,14 @@ public class CoreSchemaManager implements SchemaManager {
     private static final Logger logger = LoggerFactory.getLogger( CoreSchemaManager.class
);
 
     private final Setup setup;
-    private final CassandraFig cassandraFig;
+    private final CassandraConfig cassandraConfig;
     private final LockManager lockManager;
     private final DataStaxCluster dataStaxCluster;
 
 
     public CoreSchemaManager( final Setup setup, Injector injector ) {
         this.setup = setup;
-        this.cassandraFig = injector.getInstance( CassandraFig.class );
+        this.cassandraConfig = injector.getInstance( CassandraConfig.class );
         this.lockManager = injector.getInstance( LockManager.class );
         this.dataStaxCluster = injector.getInstance( DataStaxCluster.class );
     }
@@ -80,16 +81,21 @@ public class CoreSchemaManager implements SchemaManager {
 
     @Override
     public void destroy() {
-        logger.info( "dropping keyspaces" );
         try {
+            logger.info( "dropping application keyspace" );
             dataStaxCluster.getClusterSession()
-                .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraFig.getApplicationKeyspace()));
+                .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraConfig.getApplicationKeyspace()));
+            dataStaxCluster.waitForSchemaAgreement();
+
+            logger.info( "dropping application local keyspace" );
+            dataStaxCluster.getClusterSession()
+                .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraConfig.getApplicationLocalKeyspace()));
             dataStaxCluster.waitForSchemaAgreement();
 
             dataStaxCluster.getClusterSession().close(); // close session so it's meta will
get refreshed
         }
         catch ( Exception e ) {
-            logger.error("Error dropping application keyspace: {} error: {}", cassandraFig.getApplicationKeyspace(),
e);
+            logger.error("Error dropping application keyspaces: {} error: {}", cassandraConfig.getApplicationKeyspace(),
e);
         }
         logger.info( "keyspaces dropped" );
         logger.info( "dropping indices" );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
index d2bff37..a1ff84b 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
@@ -74,7 +74,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
     }
 
 
-    @Test( timeout = 120000 )
+    @Test( timeout = 240000 )
     public void rebuildOneCollectionIndex() throws Exception {
 
         logger.info( "Started rebuildOneCollectionIndex()" );
@@ -163,7 +163,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
 
         waitForRebuild( status, reIndexService );
 
-        app.waitForQueueDrainAndRefreshIndex(5000);
+        app.waitForQueueDrainAndRefreshIndex(15000);
 
         // ----------------- test that we can read the catherder collection and not the catshepard
 
@@ -278,7 +278,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
 
             assertNotNull( status.getJobId(), "JobId is present" );
 
-            logger.info( "Rebuilt index" );
+            logger.info( "Rebuilt index, jobID={}", status.getJobId());
 
 
             waitForRebuild( status, reIndexService );
@@ -511,14 +511,26 @@ public class RebuildIndexTest extends AbstractCoreIT {
      * Wait for the rebuild to occur
      */
     private void waitForRebuild( final ReIndexService.ReIndexStatus status, final ReIndexService
reIndexService )
-        throws InterruptedException {
+        throws InterruptedException, IllegalArgumentException {
+        if (status != null) {
+            logger.info("waitForRebuild: jobID={}", status.getJobId());
+        } else {
+            logger.info("waitForRebuild: error, status = null");
+            throw new IllegalArgumentException("reindexStatus = null");
+        }
         while ( true ) {
 
             try {
                 final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus(
status.getJobId() );
 
-                if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) {
-                    break;
+                if (updatedStatus == null) {
+                    logger.info("waitForRebuild: updated status is null");
+                } else {
+                    logger.info("waitForRebuild: status={} numberProcessed={}", updatedStatus.getStatus().toString(),
updatedStatus.getNumberProcessed());
+
+                    if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) {
+                        break;
+                    }
                 }
             }
             catch ( IllegalArgumentException iae ) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index df253f0..24f3046 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -29,7 +29,7 @@ elasticsearch.management_number_replicas=0
 elasticsearch.managment_index=usergrid_core_management
 #cassandra.keyspace.application=core_tests_schema
 elasticsearch.queue_impl.resolution=true
-elasticsearch.queue_impl=DISTRIBUTED
+elasticsearch.queue_impl=LOCAL
 
 elasticsearch.buffer_timeout=1
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index 4b76ca4..ee7b039 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -68,7 +68,7 @@ usergrid.cluster.port=2555
 collection.uniquevalues.actors=300
 collection.uniquevalues.authoritative.region=us-east
 
-elasticsearch.queue_impl=DISTRIBUTED
+elasticsearch.queue_impl=LOCAL
 
 # Queueing Test Settings
 # Reduce the long polling time for the tests

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
index c035192..9b3dfcd 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
@@ -49,9 +49,9 @@ public abstract class AbstractServiceNotificationIT extends AbstractServiceIT
{
 
     protected Notification notificationWaitForComplete(Notification notification)
             throws Exception {
-        long timeout = System.currentTimeMillis() + 60000;
+        long timeout = System.currentTimeMillis() + 120000;
         while (System.currentTimeMillis() < timeout) {
-            app.waitForQueueDrainAndRefreshIndex(200);
+            app.waitForQueueDrainAndRefreshIndex(1000);
             notification = app.getEntityManager().get(notification.getUuid(), Notification.class);
             if (notification.getFinished() != null) {
                 return notification;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index 8360009..ecc5443 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -632,7 +632,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT
{
 
 
         // receipts are created and queried, wait a bit longer for this to happen as indexing
-        app.waitForQueueDrainAndRefreshIndex(500);
+        app.waitForQueueDrainAndRefreshIndex(5000);
 
         // get the receipts entity IDs
         List<EntityRef> receipts = getNotificationReceipts(notification);

http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/usergrid-custom-test.properties b/stack/services/src/test/resources/usergrid-custom-test.properties
index bcc8b8e..4178a07 100644
--- a/stack/services/src/test/resources/usergrid-custom-test.properties
+++ b/stack/services/src/test/resources/usergrid-custom-test.properties
@@ -35,7 +35,7 @@ elasticsearch.managment_index=usergrid_services_management
 
 elasticsearch.buffer_timeout=1
 elasticsearch.queue_impl.resolution=true
-elasticsearch.queue_impl=DISTRIBUTED
+elasticsearch.queue_impl=LOCAL
 
 # Queueing Test Settings
 queue.long.polling.time.millis=50


Mime
View raw message