usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sfeld...@apache.org
Subject [49/50] incubator-usergrid git commit: Fixes bug in writeunique verify failover command
Date Mon, 16 Mar 2015 23:38:17 GMT
Fixes bug in writeunique verify failover command

Fixes NoAWSCredsRule to use Assumptions to mark as ignored

Adds AWS creds rule to NotificationsServiceTest


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/03341191
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/03341191
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/03341191

Branch: refs/heads/two-dot-o
Commit: 03341191a2109b8d297387c37b88df7af7b50218
Parents: f20dfd3
Author: Todd Nine <tnine@apigee.com>
Authored: Sun Mar 15 15:34:45 2015 -0600
Committer: Todd Nine <tnine@apigee.com>
Committed: Sun Mar 15 15:34:45 2015 -0600

----------------------------------------------------------------------
 .../corepersistence/StaleIndexCleanupTest.java  |  3 ++-
 .../impl/EntityVersionCleanupTask.java          |  7 +++--
 .../mvcc/stage/write/WriteUniqueVerify.java     | 24 +++++++++--------
 .../collection/util/EntityUtils.java            | 28 +++++++++++++++++---
 .../index/impl/EsIndexBufferConsumerImpl.java   | 14 ++++++++--
 .../persistence/queue/NoAWSCredsRule.java       | 17 ++++++++++++
 stack/services/pom.xml                          | 13 ++++++++-
 .../notifications/NotifiersServiceIT.java       |  6 +++++
 8 files changed, 90 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index ac04dcc..4bde50e 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -392,9 +392,10 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
         // wait for indexes to be cleared for the deleted entities
         count = 0;
         do {
+            queryCollectionEm("dogs", "select *");
             Thread.sleep(100);
             crs = queryCollectionCp("dogs", "dog", "select *");
-        } while ( crs.size() == numEntities && count++ < 15 );
+        } while ( crs.size() != numEntities && count++ < 15 );
 
         Assert.assertEquals("Expect candidates without earlier stale entities", crs.size(),
numEntities);
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index 03a5ff6..65506ec 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@ -28,6 +28,7 @@ import org.apache.usergrid.persistence.collection.MvccEntity;
 import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
 import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
 import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.field.Field;
 import org.slf4j.Logger;
@@ -158,10 +159,8 @@ public class EntityVersionCleanupTask implements Task<Void> {
                             final Entity entity = mvccEntity.getEntity().get();
 
                             //remove all unique fields from the index
-                            for ( final Field field : entity.getFields() ) {
-                                if ( !field.isUnique() ) {
-                                    continue;
-                                }
+                            for ( final Field field : EntityUtils.getUniqueFields(entity
)) {
+
                                 final UniqueValue unique = new UniqueValueImpl( field, entityId,
entityVersion );
                                 final MutationBatch deleteMutation =
                                     uniqueValueSerializationStrategy.delete( scope, unique
);

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 173efa7..5bdf3b9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -31,6 +31,8 @@ import com.netflix.hystrix.HystrixCommand;
 import com.netflix.hystrix.HystrixCommandGroupKey;
 import com.netflix.hystrix.HystrixThreadPoolProperties;
 import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableLifecycle;
+
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
 import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
 import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
 import org.slf4j.Logger;
@@ -108,20 +110,21 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
         //
         // Construct all the functions for verifying we're unique
         //
-        for ( final Field field :  entity.getFields() ) {
+
+
+        for ( final Field field : EntityUtils.getUniqueFields(entity)) {
 
             // if it's unique, create a function to validate it and add it to the list of
             // concurrent validations
-            if ( field.isUnique() ) {
-                // use write-first then read strategy
-                final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(),
mvccEntity.getVersion() );
 
-                // use TTL in case something goes wrong before entity is finally committed
-                final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout()
);
+            // use write-first then read strategy
+            final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion()
);
 
-                batch.mergeShallow( mb );
-                uniqueFields.add(field);
-            }
+            // use TTL in case something goes wrong before entity is finally committed
+            final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout()
);
+
+            batch.mergeShallow( mb );
+            uniqueFields.add(field);
         }
 
         //short circuit nothing to do
@@ -175,12 +178,11 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
         }
 
         public Map<String, Field> executeStrategy(ConsistencyLevel consistencyLevel){
-            Collection<Field> entityFields = entity.getFields();
             //allocate our max size, worst case
             //now get the set of fields back
             final UniqueValueSet uniqueValues;
             try {
-                uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel,
entityFields );
+                uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel,
uniqueFields );
             }
             catch ( ConnectionException e ) {
                 throw new RuntimeException( "Unable to read from cassandra", e );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
index 6233075..cf964a3 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
@@ -1,8 +1,11 @@
 package org.apache.usergrid.persistence.collection.util;
 
 
-import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
 import java.util.UUID;
+import org.apache.usergrid.persistence.model.field.Field;
 
 import org.apache.commons.lang3.reflect.FieldUtils;
 
@@ -16,9 +19,9 @@ import org.apache.usergrid.persistence.model.entity.Id;
 public class EntityUtils {
 
 
-    private static final Field VERSION = FieldUtils.getField( Entity.class, "version", true
);
+    private static final java.lang.reflect.Field VERSION = FieldUtils.getField( Entity.class,
"version", true );
 
-    private static final Field ID = FieldUtils.getField( Entity.class, "id", true );
+    private static final java.lang.reflect.Field ID = FieldUtils.getField( Entity.class,
"id", true );
 
 
     /**
@@ -46,4 +49,23 @@ public class EntityUtils {
             throw new RuntimeException( "Unable to set the field " + ID + " into the entity",
e );
         }
     }
+
+
+    /**
+     * Get the unique fields for an entity
+     * @param entity
+     * @return
+     */
+    public static List<Field> getUniqueFields( final Entity entity ){
+
+        final List<Field> uniqueFields = new ArrayList<>(entity.getFields().size());
+
+        for(final Field field: entity.getFields()){
+            if(field.isUnique()){
+                uniqueFields.add( field);
+            }
+        }
+
+        return uniqueFields;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index d55073a..b31cf39 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -322,8 +322,18 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer
{
 
         for (BulkItemResponse response : responses) {
             if (response.isFailed()) {
-                throw new RuntimeException("Unable to index documents.  Errors are :"
-                    + response.getFailure().getMessage());
+
+                final BulkItemResponse.Failure failure = response.getFailure();
+
+                final String message;
+
+                if(failure != null) {
+                    message =  "Unable to index documents.  Errors are :" + response.getFailure().getMessage();
+                }else{
+                    message =  "Unable to index documents.  Response is :" + response.getResponse();
+                }
+
+                throw new RuntimeException(message);
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
index 45218c2..ba0dc1f 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
@@ -20,6 +20,8 @@
 package org.apache.usergrid.persistence.queue;
 
 
+import org.junit.Assume;
+import org.junit.internal.runners.model.MultipleFailureException;
 import org.junit.rules.TestRule;
 import org.junit.runner.Description;
 import org.junit.runners.model.Statement;
@@ -49,6 +51,9 @@ public class NoAWSCredsRule implements TestRule {
                         throw t;
                     }
 
+                    //do this so our test gets marked as ignored.  Not pretty, but it works
+                    Assume.assumeTrue( false );
+
 
                 }
             }
@@ -69,6 +74,18 @@ public class NoAWSCredsRule implements TestRule {
             }
         }
 
+        /**
+         * Handle the multiple failure junit trace
+         */
+        if( t instanceof MultipleFailureException ){
+            for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
+                final boolean isMissingCreds = isMissingCredsException( failure );
+
+                if(isMissingCreds){
+                    return true;
+                }
+            }
+        }
         final Throwable cause = t.getCause();
 
         if ( cause == null ) {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index f5e8dd5..68f1a4a 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -133,6 +133,7 @@
       <scope>runtime</scope>
     </dependency>
 
+
     <dependency>
       <groupId>org.apache.usergrid</groupId>
       <artifactId>usergrid-core</artifactId>
@@ -290,7 +291,15 @@
       <classifier>tests</classifier>
     </dependency>
 
-    <dependency>
+      <dependency>
+          <groupId>${project.parent.groupId}</groupId>
+          <artifactId>queue</artifactId>
+          <version>${project.version}</version>
+          <type>test-jar</type>
+          <scope>test</scope>
+      </dependency>
+
+      <dependency>
       <groupId>org.apache.usergrid</groupId>
       <artifactId>usergrid-config</artifactId>
       <version>${project.version}</version>
@@ -311,6 +320,8 @@
       <scope>test</scope>
     </dependency>
 
+
+
     <dependency>
       <groupId>org.springframework</groupId>
       <artifactId>spring-test</artifactId>

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
b/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
index 3c1c645..e6970b5 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
@@ -17,12 +17,15 @@
 package org.apache.usergrid.services.notifications;
 
 import org.apache.commons.io.IOUtils;
+
+import org.apache.usergrid.persistence.queue.NoAWSCredsRule;
 import org.apache.usergrid.services.notifications.apns.MockSuccessfulProviderAdapter;
 import org.apache.usergrid.persistence.entities.Notifier;
 
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
+import org.junit.Rule;
 import org.junit.Test;
 import org.apache.usergrid.services.notifications.ConnectionException;
 import org.apache.usergrid.services.notifications.NotificationsService;
@@ -48,6 +51,9 @@ public class NotifiersServiceIT extends AbstractServiceIT {
 
     private QueueListener listener;
 
+    @Rule
+    public NoAWSCredsRule noCredsRule = new NoAWSCredsRule();
+
 
     @Before
     public void before() throws Exception {


Mime
View raw message