usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mdun...@apache.org
Subject usergrid git commit: Fixed failing tests and bugs -Fixed tests for Core module that were broken due to changes in 8913e43 -Fixed failing tests for Services module related to threading issues -Fixed failing tests for Rest module related to Index rebuild -
Date Wed, 05 Sep 2018 06:38:59 GMT
Repository: usergrid
Updated Branches:
  refs/heads/master 8913e43d2 -> faafed885


Fixed failing tests and bugs
-Fixed tests for Core module that were broken due to changes in 8913e43
-Fixed failing tests for Services module related to threading issues
-Fixed failing tests for Rest module related to Index rebuild
-Fixed re-index code to ensure rebuild status is correctly maintained
-Updated Shiro to 1.3.2 (latest)


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/faafed88
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/faafed88
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/faafed88

Branch: refs/heads/master
Commit: faafed8858d9999195337f1776a94d57f84ac7be
Parents: 8913e43
Author: Keyur Karnik <keyur.karnik+github@gmail.com>
Authored: Wed Aug 29 05:00:13 2018 -0700
Committer: Keyur Karnik <keyur.karnik+github@gmail.com>
Committed: Tue Sep 4 21:25:31 2018 -0700

----------------------------------------------------------------------
 .../corepersistence/index/ReIndexService.java   |  2 +-
 .../index/ReIndexServiceImpl.java               | 22 ++++----
 .../usergrid/persistence/RebuildIndexTest.java  | 57 ++++++++++++++++----
 stack/pom.xml                                   |  2 +-
 .../apache/usergrid/rest/IndexResourceIT.java   | 45 +++++++++++-----
 .../org/apache/usergrid/NewOrgAppAdminRule.java |  1 +
 .../management/OrganizationConfigIT.java        |  4 +-
 .../management/export/ExportServiceIT.java      |  3 ++
 8 files changed, 98 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index d37f117..48c3908 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -97,7 +97,7 @@ public interface ReIndexService {
         }
 
         /**
-         * Get the jobId used to resume this operation
+         * Get the collectionName used to resume this operation
          */
         public String getCollectionName() {
             return collectionName;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index d4fb249..036f89c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -202,7 +202,7 @@ public class ReIndexServiceImpl implements ReIndexService {
             .subscribeOn( Schedulers.io() ).subscribe();
 
         if(isForCollection){
-            return new ReIndexStatus( "", Status.STARTED, 0, 0, reIndexRequestBuilder.getCollectionName().get()
);
+            return new ReIndexStatus( "", Status.STARTED, 0, 0, CpNamingUtils.getNameFromEdgeType(reIndexRequestBuilder.getCollectionName().get())
);
 
         }
 
@@ -365,21 +365,23 @@ public class ReIndexServiceImpl implements ReIndexService {
     private void writeStateMetaForCollection(final String appIdString, final String collectionName,
                                              final Status status, final long processedCount,
final long lastUpdated ) {
 
-        if(logger.isDebugEnabled()) {
+    	String prefixedColName = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName.toLowerCase()
);
+    	if(logger.isDebugEnabled()) {
             logger.debug( "Flushing state for collection {}, status {}, processedCount {},
lastUpdated {}",
-                collectionName, status, processedCount, lastUpdated);
+            		collectionName, status, processedCount, lastUpdated);
         }
 
-        mapManager.putString( appIdString + MAP_SEPARATOR + collectionName + MAP_STATUS_KEY,
status.name() );
-        mapManager.putLong( appIdString + MAP_SEPARATOR + collectionName + MAP_COUNT_KEY,
processedCount );
-        mapManager.putLong( appIdString + MAP_SEPARATOR + collectionName + MAP_UPDATED_KEY,
lastUpdated );
+        mapManager.putString( appIdString + MAP_SEPARATOR + prefixedColName + MAP_STATUS_KEY,
status.name() );
+        mapManager.putLong( appIdString + MAP_SEPARATOR + prefixedColName + MAP_COUNT_KEY,
processedCount );
+        mapManager.putLong( appIdString + MAP_SEPARATOR + prefixedColName + MAP_UPDATED_KEY,
lastUpdated );
     }
 
 
     private ReIndexStatus getIndexResponseForCollection( final String appIdString, final
String collectionName ) {
 
-        final String stringStatus =
-            mapManager.getString( appIdString + MAP_SEPARATOR + collectionName + MAP_STATUS_KEY
);
+        String prefixedColName = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName.toLowerCase()
);
+    	final String stringStatus =
+            mapManager.getString( appIdString + MAP_SEPARATOR + prefixedColName + MAP_STATUS_KEY
);
 
         if(stringStatus == null){
             return new ReIndexStatus( "", Status.UNKNOWN, 0, 0, collectionName );
@@ -387,8 +389,8 @@ public class ReIndexServiceImpl implements ReIndexService {
 
         final Status status = Status.valueOf( stringStatus );
 
-        final long processedCount = mapManager.getLong( appIdString + MAP_SEPARATOR + collectionName
+ MAP_COUNT_KEY );
-        final long lastUpdated = mapManager.getLong( appIdString + MAP_SEPARATOR + collectionName
+ MAP_UPDATED_KEY );
+        final long processedCount = mapManager.getLong( appIdString + MAP_SEPARATOR + prefixedColName
+ MAP_COUNT_KEY );
+        final long lastUpdated = mapManager.getLong( appIdString + MAP_SEPARATOR + prefixedColName
+ MAP_UPDATED_KEY );
 
         return new ReIndexStatus( "", status, processedCount, lastUpdated, collectionName
);
     }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
index 57962c0..90af5ba 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
@@ -154,12 +154,12 @@ public class RebuildIndexTest extends AbstractCoreIT {
 
         ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex(builder);
 
-        assertNotNull(status.getJobId(), "JobId is present");
+        assertNotNull(status.getCollectionName(), "Collection name is present");
 
         logger.info("Rebuilt index");
 
 
-        waitForRebuild(status, reIndexService);
+        waitForRebuild(em.getApplicationId().toString(), status.getCollectionName(), reIndexService);
 
         //app.waitForQueueDrainAndRefreshIndex(15000);
 
@@ -279,7 +279,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
             logger.info("Rebuilt index, jobID={}", status.getJobId());
 
 
-            waitForRebuild(status, reIndexService);
+            waitForRebuild(status.getJobId(), reIndexService);
 
 
             logger.info("Rebuilt index");
@@ -387,7 +387,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
 
             logger.info("Rebuilt index");
 
-            waitForRebuild(status, reIndexService);
+            waitForRebuild(status.getJobId(), reIndexService);
 
             logger.info("Rebuilt index");
 
@@ -486,7 +486,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
 
             logger.info("Rebuilt index");
 
-            waitForRebuild(status, reIndexService);
+            waitForRebuild(status.getJobId(), reIndexService);
 
             logger.info("Rebuilt index");
 
@@ -505,18 +505,53 @@ public class RebuildIndexTest extends AbstractCoreIT {
     /**
      * Wait for the rebuild to occur
      */
-    private void waitForRebuild(final ReIndexService.ReIndexStatus status, final ReIndexService
reIndexService)
+    private void waitForRebuild(final String jobId, final ReIndexService reIndexService)
         throws InterruptedException, IllegalArgumentException {
-        if (status != null) {
-            logger.info("waitForRebuild: jobID={}", status.getJobId());
+        if (jobId != null && !jobId.trim().equals("")) {
+            logger.info("waitForRebuild: jobID={}", jobId);
         } else {
-            logger.info("waitForRebuild: error, status = null");
-            throw new IllegalArgumentException("reindexStatus = null");
+            logger.info("waitForRebuild: error, jobId = null or empty");
+            throw new IllegalArgumentException("jobId = null or empty");
         }
         while (true) {
 
             try {
-                final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus(status.getJobId());
+                final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus(jobId);
+
+                if (updatedStatus == null) {
+                    logger.info("waitForRebuild: updated status is null");
+                } else {
+                    logger.info("waitForRebuild: status={} numberProcessed={}", updatedStatus.getStatus().toString(),
updatedStatus.getNumberProcessed());
+
+                    if (updatedStatus.getStatus() == ReIndexService.Status.COMPLETE) {
+                        break;
+                    }
+                }
+            } catch (IllegalArgumentException iae) {
+                //swallow.  Thrown if our job can't be found.  I.E hasn't updated yet
+            }
+
+
+            Thread.sleep(1000);
+        }
+    }
+    
+    
+    /**
+     * Wait for the rebuild to occur
+     */
+    private void waitForRebuild(final String appId, final String collectionName, final ReIndexService
reIndexService)
+        throws InterruptedException, IllegalArgumentException {
+        if (appId != null && !appId.trim().equals("") && collectionName !=
null && !collectionName.trim().equals("")) {
+            logger.info("waitForRebuild: appId={} collName={}", appId, collectionName);
+        } else {
+            logger.info("waitForRebuild: error, appId or collName = null or empty");
+            throw new IllegalArgumentException("appId or collName = null or empty");
+        }
+        while (true) {
+
+            try {
+                final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatusForCollection(appId,
collectionName);
 
                 if (updatedStatus == null) {
                     logger.info("waitForRebuild: updated status is null");

http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index c98c72d..aa289d0 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -114,7 +114,7 @@
         <junit-version>4.12</junit-version>
         <log4j-version>1.2.16</log4j-version>
         <org.springframework.version>3.2.13.RELEASE</org.springframework.version>
-        <shiro-version>1.2.4</shiro-version>
+        <shiro-version>1.3.2</shiro-version>
         <slf4j-version>1.7.2</slf4j-version>
         <snakeyaml-version>1.9</snakeyaml-version>
         <tomcat-version>7.0.64</tomcat-version>

http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
index 28d6501..be71881 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
@@ -25,6 +25,7 @@ import org.apache.usergrid.rest.test.resource.AbstractRestIT;
 import org.apache.usergrid.rest.test.resource.model.ApiResponse;
 import org.apache.usergrid.rest.test.resource.model.QueryParameters;
 import org.apache.usergrid.rest.test.resource.model.Token;
+import org.apache.usergrid.corepersistence.index.ReIndexService.Status;
 import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -68,35 +69,51 @@ public class IndexResourceIT extends AbstractRestIT {
             .get(clientSetup.getSuperuserName(),clientSetup.getSuperuserPassword());
 
         QueryParameters queryParameters = new QueryParameters();
-        queryParameters.addParam( "access_token",superUserToken.getAccessToken());
+        queryParameters.addParam("access_token", superUserToken.getAccessToken());
         ApiResponse result = clientSetup.getRestClient()
-            .pathResource( "system/index/rebuild/" + clientSetup.getAppUuid() + "/StOrElaTloNs"
)
-            .post( false, ApiResponse.class, null, queryParameters, true );
+            .pathResource("system/index/rebuild/" + clientSetup.getAppUuid() + "/StOrElaTloNs")
+            .post(false, ApiResponse.class, null, queryParameters, true);
 
         assertNotNull(result);
+        assertEquals(Status.STARTED.name(), result.getStatus());
 
         //try the reindex endpoint with all lowercase Characters
         queryParameters = new QueryParameters();
-        queryParameters.addParam( "access_token",clientSetup.getSuperuserToken().getAccessToken()
);
+        queryParameters.addParam("access_token", superUserToken.getAccessToken());
         result = clientSetup.getRestClient()
-            .pathResource( "system/index/rebuild/"+clientSetup.getAppUuid()+"/storelatlons"
)
-            .post( false, ApiResponse.class,null,queryParameters,true);
-        String status = result.getProperties().get("jobId").toString();
-
-        assertNotNull( result );
+            .pathResource("system/index/rebuild/" + clientSetup.getAppUuid() + "/storelatlons")
+            .post(false, ApiResponse.class, null, queryParameters, true);
+        
+        assertNotNull(result);
+        //at this point, this could return a result of the previous reindex, or if it has
completed, it will create a new job
+        assertNotEquals(Status.UNKNOWN.name(), result.getStatus());
 
         WebTarget res = clientSetup.getRestClient()
-            .pathResource( "system/index/rebuild/" + result.getProperties()
-                .get( "jobId" ).toString() )
+            .pathResource("system/index/rebuild/" + clientSetup.getAppUuid() + "/storelatlons")
             .getTarget();
 
         HttpAuthenticationFeature feature = HttpAuthenticationFeature.basicBuilder()
             .credentials( "superuser", "superpassword" ).build();
 
         result = res.register( feature ).request().get( ApiResponse.class );
-
-        assertNotNull( result );
-        assertEquals(status,result.getProperties().get("jobId").toString());
+        assertNotNull(result);
+        
+        int retry = 0;
+        while(retry < 5 && !result.getStatus().equals(Status.COMPLETE.name()))
{
+        	try {
+        		//hope reindex completes, if not, that's still ok
+				Thread.sleep(1000);
+			} catch (InterruptedException e) {
+				
+			}
+        	result = res.register( feature ).request().get( ApiResponse.class );
+        	retry++;
+        	assertNotNull(result);
+        }
+        
+        Map<String, Object> resultMap = result.getProperties();
+        assertNotNull( resultMap );
+        assertEquals(1,resultMap.get("numberQueued"));
 
 
     }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java b/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java
index 659520e..892a440 100644
--- a/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java
+++ b/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java
@@ -120,6 +120,7 @@ public class NewOrgAppAdminRule implements TestRule {
      * Create the org admin and application
      */
     protected void before( Description description ) throws Exception {
+    	logger.info( "Test {}: Starting with application", description.getDisplayName() );
         final String className = description.getClassName();
         final String methodName = description.getMethodName();
         final String uuidString = newUUIDString();

http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java
b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java
index 6424dbd..d2932c2 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java
@@ -32,12 +32,14 @@ import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
 
+import net.jcip.annotations.NotThreadSafe;
+
 import java.util.*;
 
 import static org.apache.usergrid.TestHelper.*;
 import static org.junit.Assert.*;
 
-
+@NotThreadSafe
 public class OrganizationConfigIT {
 
     @Rule

http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
index 870b678..2268ea6 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
@@ -59,6 +59,8 @@ import com.amazonaws.SDKGlobalConfiguration;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Module;
 
+import net.jcip.annotations.NotThreadSafe;
+
 import static org.apache.usergrid.TestHelper.newUUIDString;
 import static org.apache.usergrid.TestHelper.uniqueApp;
 import static org.apache.usergrid.TestHelper.uniqueOrg;
@@ -74,6 +76,7 @@ import static org.mockito.Mockito.when;
  *
  *
  */
+@NotThreadSafe
 public class ExportServiceIT {
 
     private static final Logger logger = LoggerFactory.getLogger( ExportServiceIT.class );


Mime
View raw message