usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mru...@apache.org
Subject [1/5] usergrid git commit: Disable indexing per Entity type for USERGRID-1278. https://issues.apache.org/jira/browse/USERGRID-1278
Date Thu, 14 Apr 2016 23:01:19 GMT
Repository: usergrid
Updated Branches:
  refs/heads/release-2.1.1 2e576f373 -> 8dce5ed8c


Disable indexing per Entity type for USERGRID-1278. https://issues.apache.org/jira/browse/USERGRID-1278


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fbced588
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fbced588
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fbced588

Branch: refs/heads/release-2.1.1
Commit: fbced588bdf7de5f7192d20e2da6a8740df0218a
Parents: 2e576f3
Author: Dave Johnson <snoopdave@apache.org>
Authored: Thu Apr 14 12:26:06 2016 -0400
Committer: Dave Johnson <snoopdave@apache.org>
Committed: Thu Apr 14 12:26:06 2016 -0400

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        | 86 ++++++++++++++------
 .../collection/CollectionsResourceIT.java       | 56 +++++++++++--
 2 files changed, 112 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/fbced588/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 891c1fe..b2330f3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -601,14 +601,36 @@ public class CpEntityManager implements EntityManager {
             handleWriteUniqueVerifyException( entity, wuve );
         }
 
-        // queue an event to update the new entity
-        indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
+        if ( !skipIndexingForType( cpEntity.getId().getType() ) ) {
 
+            // queue an event to update the new entity
+            indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
 
-        // queue up an event to clean-up older versions than this one from the index
-        if(entityManagerFig.getDeindexOnUpdate()) {
-            indexService.queueDeIndexOldVersion(applicationScope, entityId);
+            // queue up an event to clean-up older versions than this one from the index
+            if (entityManagerFig.getDeindexOnUpdate()) {
+                indexService.queueDeIndexOldVersion( applicationScope, entityId );
+            }
+        }
+    }
+
+    private boolean skipIndexingForType( String type ) {
+
+        boolean skipIndexing = false;
+
+        MapManager mm = getMapManagerForTypes();
+        IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm );
+        String collectionName = Schema.defaultCollectionName( type );
+        Optional<Map> collectionIndexingSchema =  indexSchemaCache.getCollectionSchema(
collectionName );
+
+        if ( collectionIndexingSchema.isPresent()) {
+            Map jsonMapData = collectionIndexingSchema.get();
+            final ArrayList fields = (ArrayList) jsonMapData.get( "fields" );
+            if ( fields.size() == 1 && fields.get(0).equals("!")) {
+                skipIndexing = true;
+            }
         }
+
+        return skipIndexing;
     }
 
 
@@ -670,8 +692,9 @@ public class CpEntityManager implements EntityManager {
 
         Id entityId = new SimpleId( entityRef.getUuid(), entityRef.getType() );
 
-        //Step 4 && 5
-        indexService.queueEntityDelete( applicationScope, entityId );
+        if ( !skipIndexingForType( entityId.getType() ) ) {
+            indexService.queueEntityDelete( applicationScope, entityId );
+        }
 
         //Step 6
         //delete from our UUID index
@@ -714,7 +737,8 @@ public class CpEntityManager implements EntityManager {
     }
 
     @Override
-    public Results searchCollectionConsistent( EntityRef entityRef, String collectionName,
Query query, int expectedResults) throws Exception {
+    public Results searchCollectionConsistent(
+        EntityRef entityRef, String collectionName, Query query, int expectedResults) throws
Exception {
 
         return getRelationManager( entityRef ).searchCollectionConsistent(collectionName,
query, expectedResults);
     }
@@ -754,8 +778,8 @@ public class CpEntityManager implements EntityManager {
     public RelationManager getRelationManager( EntityRef entityRef ) {
         Preconditions.checkNotNull(entityRef, "entityRef cannot be null");
 
-        CpRelationManager relationManager =
-            new CpRelationManager(managerCache, indexService, collectionService, connectionService,
this, entityManagerFig, applicationId, entityRef );
+        CpRelationManager relationManager = new CpRelationManager( managerCache, indexService,
collectionService,
+            connectionService, this, entityManagerFig, applicationId, entityRef );
         return relationManager;
     }
 
@@ -830,7 +854,8 @@ public class CpEntityManager implements EntityManager {
 
         Timer.Context repairedEntityGet = entGetRepairedEntityTimer.time();
 
-        //TODO: can't we just sub in the getEntityRepair method here so for every read of
a uniqueEntityField we can verify it is correct?
+        // TODO: can't we just sub in the getEntityRepair method here
+        // so for every read of a uniqueEntityField we can verify it is correct?
 
         StringField uniqueLookupRepairField =  new StringField( propertyName, aliasType.toString());
 
@@ -916,8 +941,8 @@ public class CpEntityManager implements EntityManager {
         // add a warn statement so we can see if we have data migration issues.
         // TODO When we get an event system, trigger a repair if this is detected
         if ( results.size() > 1 ) {
-            logger.warn( "More than 1 entity with Owner id '{}' of type '{}' and alias '{}'
exists. This is a duplicate alias, and needs audited",
-                    ownerRef, collectionType, aliasValue );
+            logger.warn( "More than 1 entity with Owner id '{}' of type '{}' and alias '{}'
exists. " +
+                "This is a duplicate alias, and needs audited", ownerRef, collectionType,
aliasValue );
         }
 
         return results.get(aliasValue);
@@ -1139,7 +1164,9 @@ public class CpEntityManager implements EntityManager {
 
         //Adding graphite metrics
 
-        indexService.queueEntityIndexUpdate(applicationScope, cpEntity, 0);
+        if ( !skipIndexingForType( cpEntity.getId().getType() ) ) {
+            indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
+        }
     }
 
 
@@ -1785,12 +1812,17 @@ public class CpEntityManager implements EntityManager {
 
         //do a check to see if you have a * field. If you do have a * field then ignore all
other fields
         //and only accept the * field.
-        if(fieldProperties.contains( "*" )){
-            ArrayList<String> wildCardArrayList = new ArrayList<>(  );
+        if ( fieldProperties.contains( "*" )) {
+            ArrayList<String> wildCardArrayList = new ArrayList<>();
             wildCardArrayList.add( "*" );
-            schemaMap.put( "fields",wildCardArrayList );
-        }
-        else {
+            schemaMap.put( "fields", wildCardArrayList );
+
+        } else if ( fieldProperties.contains( "!" )) {
+            ArrayList<String> wildCardArrayList = new ArrayList<>();
+            wildCardArrayList.add( "!" );
+            schemaMap.put( "fields", wildCardArrayList );
+
+        } else {
             schemaMap.putAll( properties );
         }
 
@@ -1815,7 +1847,7 @@ public class CpEntityManager implements EntityManager {
     public Object getCollectionSchema( String collectionName ){
         MapManager mm = getMapManagerForTypes();
 
-        IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm ); //managerCache.getIndexSchema(
mm );
+        IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm );
 
         Optional<Map> collectionIndexingSchema =  indexSchemaCache.getCollectionSchema(
collectionName );
 
@@ -2388,7 +2420,8 @@ public class CpEntityManager implements EntityManager {
         if ( !skipAggregateCounters ) {
             long timestamp = cass.createTimestamp();
             Mutator<ByteBuffer> m = createMutator( cass.getApplicationKeyspace( applicationId
), be );
-            counterUtils.batchIncrementAggregateCounters( m, applicationId, userId, groupId,
null, category, counters, timestamp );
+            counterUtils.batchIncrementAggregateCounters(
+                m, applicationId, userId, groupId, null, category, counters, timestamp );
 
             //Adding graphite metrics
             Timer.Context timeIncrementCounters =entIncrementAggregateCountersTimer.time();
@@ -2987,8 +3020,9 @@ public class CpEntityManager implements EntityManager {
 
         final SearchEdgeType searchByEdgeType = createConnectionTypeSearch( entityRef.asId()
);
 
-        return graphManager.getEdgeTypesFromSource( searchByEdgeType ).map( edgeName ->
getConnectionNameFromEdgeName( edgeName ) ).collect(
-            () -> new HashSet<String>(), ( r, s ) -> r.add( s ) ).toBlocking().last();
+        return graphManager.getEdgeTypesFromSource(
+            searchByEdgeType ).map( edgeName -> getConnectionNameFromEdgeName( edgeName
) )
+                .collect( () -> new HashSet<String>(), ( r, s ) -> r.add( s )
).toBlocking().last();
     }
 
 
@@ -3001,7 +3035,8 @@ public class CpEntityManager implements EntityManager {
         final SearchEdgeType searchByEdgeType = createConnectionTypeSearch( entityRef.asId()
);
 
         return graphManager.getEdgeTypesToTarget(searchByEdgeType).map(
-                    edgeName -> getConnectionNameFromEdgeName( edgeName ) ).collect( ()
-> new HashSet<String>(  ), ( r, s ) -> r.add(s) ).toBlocking().last();
+            edgeName -> getConnectionNameFromEdgeName( edgeName ) )
+                .collect( () -> new HashSet<String>(  ), ( r, s ) -> r.add(s)
).toBlocking().last();
     }
 
 
@@ -3030,7 +3065,8 @@ public class CpEntityManager implements EntityManager {
             try {
                 for (int i = 0; i < 20; i++) {
                     if (searchCollection(
-                        new SimpleEntityRef(org.apache.usergrid.persistence.entities.Application.ENTITY_TYPE,
getApplicationId()),
+                        new SimpleEntityRef(
+                            org.apache.usergrid.persistence.entities.Application.ENTITY_TYPE,
getApplicationId()),
                         InflectionUtils.pluralize("refresh"),
                         Query.fromQL("select * where uuid='" + refreshEntity.getUuid() +
"'")
                     ).size() > 0

http://git-wip-us.apache.org/repos/asf/usergrid/blob/fbced588/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index d052564..b240629 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.rest.applications.collection;
 
 import com.fasterxml.jackson.databind.JsonNode;
 
+import org.apache.commons.lang.RandomStringUtils;
 import org.apache.usergrid.persistence.Schema;
 import org.apache.usergrid.persistence.entities.Application;
 
@@ -42,11 +43,7 @@ import javax.ws.rs.client.Invocation;
 import javax.ws.rs.core.MediaType;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.ArrayList;
+import java.util.*;
 
 import org.apache.commons.lang.NullArgumentException;
 
@@ -943,4 +940,53 @@ public class CollectionsResourceIT extends AbstractRestIT {
         Assert.assertEquals( "Should not be more than one name property", -1, secondFred
);
   */
     }
+
+
+    /**
+     * Test that when schema is "!" entity gets saved but does not get indexed
+     */
+    @Test
+    public void postCollectionSchemaWithWildcardIndexNone() throws Exception {
+
+        // creating schema with no index wildcard and other fields that should be ignored
+        ArrayList<String> indexingArray = new ArrayList<>(  );
+        indexingArray.add( "!" );
+        indexingArray.add( "one" );
+        indexingArray.add( "two" );
+        Entity payload = new Entity();
+        payload.put( "fields", indexingArray);
+
+        String randomizer = RandomStringUtils.randomAlphanumeric(10);
+        String collectionName = "col_" + randomizer;
+        app().collection( collectionName ).collection( "_indexes" ).post( payload );
+        refreshIndex();
+
+        // was the no-index wildcard saved and others ignored?
+        Collection collection = app().collection( collectionName ).collection( "_index" ).get();
+        LinkedHashMap testCollectionSchema = (LinkedHashMap)collection.getResponse().getData();
+        ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get(
"fields" );
+        assertTrue( schema.contains( "!" ) );
+        assertFalse( schema.contains( "one" ) );
+        assertFalse( schema.contains( "two" ) );
+
+        // post an entity with a name and a color
+        String entityName = "name_" + randomizer;
+        Entity postedEntity = this.app().collection( collectionName )
+            .post( new Entity().withProp("name", entityName).withProp( "color", "magenta"
) );
+
+        // should be able to get entity by ID
+        Entity getByIdEntity = app().collection( collectionName ).entity( postedEntity.getUuid()
).get();
+        assertNotNull( getByIdEntity );
+        assertEquals( postedEntity.getUuid(), getByIdEntity.getUuid() );
+
+        // should be able to get entity by name
+        Entity getByNameEntity = app().collection( collectionName ).entity( entityName ).get();
+        assertNotNull( getByNameEntity );
+        assertEquals( postedEntity.getUuid(), getByNameEntity.getUuid() );
+
+        // should NOT be able to get entity by query
+        Iterator<Entity> getByQuery = app().collection( collectionName )
+            .get( new QueryParameters().setQuery( "select * where color='magenta'" ) ).iterator();
+        assertFalse( getByQuery.hasNext() );
+    }
 }


Mime
View raw message