usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject [07/36] usergrid git commit: no message
Date Fri, 08 Apr 2016 20:20:54 GMT
no message


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f0480b48
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f0480b48
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f0480b48

Branch: refs/heads/release-2.1.1
Commit: f0480b4881a097e50ad2153e735268834b40dbb9
Parents: a2ec1df
Author: George Reyes <grey@apache.org>
Authored: Wed Mar 16 11:38:10 2016 -0700
Committer: George Reyes <grey@apache.org>
Committed: Thu Mar 24 09:14:15 2016 -0700

----------------------------------------------------------------------
 .../corepersistence/index/IndexServiceImpl.java | 101 +++++++++++--------
 .../persistence/index/EntityIndexBatch.java     |   3 +
 .../index/impl/EntityToMapConverter.java        |   7 ++
 .../index/impl/EsEntityIndexBatchImpl.java      |  27 ++++-
 .../persistence/index/impl/IndexOperation.java  |  19 +++-
 .../collection/CollectionsResourceIT.java       | 101 ++++++++++++++++++-
 6 files changed, 213 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0480b48/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 01bc516..9e0e799 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.corepersistence.index;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
@@ -30,6 +31,8 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.usergrid.persistence.Schema;
 import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.impl.EntityField;
+import org.apache.usergrid.persistence.index.impl.IndexOperation;
 import org.apache.usergrid.persistence.map.MapManager;
 import org.apache.usergrid.persistence.map.MapManagerFactory;
 import org.apache.usergrid.persistence.map.MapScope;
@@ -158,65 +161,83 @@ public class IndexServiceImpl implements IndexService {
 
             final EntityIndexBatch batch = ei.createBatch();
 
-           // if (logger.isDebugEnabled()) {
+            if (logger.isDebugEnabled()) {
                 logger.debug("adding edge {} to batch for entity {}", indexEdge, entity);
-           // }
+            }
 
+            Map<String, Object> map = getFilteredStringObjectMap( applicationScope,
entity, indexEdge );
 
-                indexEdge.getNodeId().getUuid();
+            if(map!=null){
+                batch.index( indexEdge, entity ,map);
+            }
+            else{
+                batch.index( indexEdge,entity );
+            }
 
-                //System.out.println("hello");
 
-                Id mapOwner = new SimpleId( indexEdge.getNodeId().getUuid(), TYPE_APPLICATION
);
+            return batch.build();
+        } );
 
-                final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner );
+        return ObservableTimer.time( batches, addTimer  );
 
-                MapManager mm = mapManagerFactory.createMapManager( ms );
 
+    }
 
 
-                String jsonMap = mm.getString( indexEdge.getEdgeName().split( "\\|" )[1]
);
+    private Map<String, Object> getFilteredStringObjectMap( final ApplicationScope
applicationScope, final Entity entity,
+                                                    final IndexEdge indexEdge ) {IndexOperation
indexOperation = new IndexOperation(  );
 
-                Set<String> defaultProperties = null;
 
-                if(jsonMap != null) {
+        indexEdge.getNodeId().getUuid();
 
-                    Map jsonMapData = ( Map ) JsonUtils.parse( jsonMap );
-                    Schema schema = Schema.getDefaultSchema();
-                    defaultProperties = schema.getRequiredProperties( indexEdge.getEdgeName().split(
"\\|" )[1]);
-                    //TODO: additional logic to
-                    ArrayList fieldsToKeep = ( ArrayList ) jsonMapData.get( "fields" );
-                    defaultProperties.addAll( fieldsToKeep );
+        Id mapOwner = new SimpleId( indexEdge.getNodeId().getUuid(), TYPE_APPLICATION );
 
-                }
+        final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner );
 
-            Entity filteredEntity = new Entity( entity.getId(),entity.getVersion() );
-            filteredEntity.setFieldMap( entity.getFieldMap() );
-
-                Collection<String> trimmedFields = null;
-                if(defaultProperties!=null) {
-                    // if(cpHeadEntity.getFields())
-                    final Set<String> finalDefaultProperties = defaultProperties;
-                    trimmedFields = entity.getFieldMap().keySet();
-                    Iterator collectionIterator = trimmedFields.iterator();
-                    while ( collectionIterator.hasNext() ) {
-                        String fieldName = ( String ) collectionIterator.next();
-                        if ( !finalDefaultProperties.contains( fieldName ) ) {
-                            //collectionIterator.remove();
-                            filteredEntity.removeField( fieldName );
-                        }
-                    }
-                }
+        MapManager mm = mapManagerFactory.createMapManager( ms );
 
+        String jsonMap = mm.getString( indexEdge.getEdgeName().split( "\\|" )[1] );
 
-            batch.index( indexEdge, filteredEntity );
-
-            return batch.build();
-        } );
-
-        return ObservableTimer.time( batches, addTimer  );
+        Set<String> defaultProperties = null;
 
+        if(jsonMap != null) {
 
+            Map jsonMapData = ( Map ) JsonUtils.parse( jsonMap );
+            Schema schema = Schema.getDefaultSchema();
+            defaultProperties = schema.getRequiredProperties( indexEdge.getEdgeName().split(
"\\|" )[1]);
+            //TODO: additional logic to
+            ArrayList fieldsToKeep = ( ArrayList ) jsonMapData.get( "fields" );
+            defaultProperties.addAll( fieldsToKeep );
+        }
+        else
+            return null;
+
+        Map<String, Object> map = indexOperation.convertedEntityToBeIndexed( applicationScope,indexEdge,entity
);
+        HashSet mapFields = ( HashSet ) map.get( "fields" );
+
+        if(defaultProperties!=null) {
+            final Set<String> finalDefaultProperties = defaultProperties;
+            Iterator collectionIterator = mapFields.iterator();
+            while ( collectionIterator.hasNext() ) {
+                EntityField testedField = ( EntityField ) collectionIterator.next();
+                String fieldName = ( String ) (testedField).get( "name" );
+                //TODO: need to change the logic here such that default properties does a
check against each every field.
+                //it is very likely that the below does O(n) comparisons anyways so doing
it for each value in a loop
+                //would be the right equivalent here. Even if it isn't very efficient.
+                //maybe if it was a hashmap, and if the hashmap didn't contain the value
then we remove the field.
+                //but now for each hash value we have to do a check to see it doesn't start
with a certain value.
+                //making this an O(n) operation anyways.
+
+                //so for each value in the finalDefaultProperties check to see if it matches
the current property
+                //then if it doesn't do a split of the properties and see if any of them
match for the split
+                //if they do match then allow in, otherwise don't allow.
+
+                if ( !finalDefaultProperties.contains( fieldName ) ) {
+                    mapFields.remove( testedField );
+                }
+            }
+        }
+        return map;
     }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0480b48/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
index 17dd4d3..e3a40f6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.persistence.index;/*
 
 
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
 
 import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
@@ -53,6 +54,8 @@ public interface EntityIndexBatch {
      */
     EntityIndexBatch deindex( final SearchEdge searchEdge, final CandidateResult result );
 
+    EntityIndexBatch index( IndexEdge indexEdge, Entity entity, Map flattenedEntityMap );
+
     /**
      * Remove index of entity.
      *

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0480b48/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
index d0a5c44..77a8d92 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
@@ -17,16 +17,22 @@
 package org.apache.usergrid.persistence.index.impl;
 
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.index.IndexEdge;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapScope;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.EntityMap;
 import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
 
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.APPLICATION_ID_FIELDNAME;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.EDGE_NAME_FIELDNAME;
@@ -45,6 +51,7 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.getType;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.nodeId;
 
 
+
 /**
  * Convert a CP entity to an elasticsearch document
  */

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0480b48/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index f703e32..5296978 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -19,6 +19,7 @@
 package org.apache.usergrid.persistence.index.impl;
 
 
+import java.util.Map;
 import java.util.UUID;
 
 import org.apache.usergrid.persistence.index.*;
@@ -31,6 +32,7 @@ import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
 import org.apache.usergrid.persistence.map.MapManager;
 import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
 
 
 public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@ -69,16 +71,35 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         final String writeAlias = alias.getWriteAlias();
 
-        //if ( logger.isDebugEnabled() ) {
-            logger.info( "Indexing to alias {} with scope {} on edge {} with entity data
{}",
+        if ( logger.isDebugEnabled() ) {
+            logger.debug( "Indexing to alias {} with scope {} on edge {} with entity data
{}",
                     writeAlias, applicationScope, indexEdge, entity.getFieldMap().keySet()
);
-        //}
+        }
 
         //add app id for indexing
         container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, indexEdge,
entity));
         return this;
     }
 
+    @Override
+    public EntityIndexBatch index ( final IndexEdge indexEdge, final Entity entity ,final
Map flattenedEntityMap){
+        IndexValidationUtils.validateIndexEdge(indexEdge);
+        ValidationUtils.verifyEntityWrite(entity);
+        ValidationUtils.verifyVersion( entity.getVersion() );
+
+        final String writeAlias = alias.getWriteAlias();
+
+        if ( logger.isDebugEnabled() ) {
+            logger.debug( "Indexing to alias {} with scope {} on edge {} with entity data
{}",
+                writeAlias, applicationScope, indexEdge, flattenedEntityMap );
+        }
+
+        //add app id for indexing
+        container.addIndexRequest(new IndexOperation(writeAlias, applicationScope,
+            indexEdge,entity.getId(),entity.getVersion(),flattenedEntityMap) );
+        return this;
+    }
+
 
     @Override
     public EntityIndexBatch deindex( final SearchEdge searchEdge, final Id id, final UUID
version ) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0480b48/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
index 28f2e0d..b51d0c7 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
@@ -21,10 +21,12 @@ package org.apache.usergrid.persistence.index.impl;
 
 
 import java.util.Map;
+import java.util.UUID;
 
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.index.IndexEdge;
 import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
 
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -48,10 +50,21 @@ public class IndexOperation implements BatchOperation {
 
     public IndexOperation( final String writeAlias, final ApplicationScope applicationScope,
IndexEdge indexEdge,
                            Entity entity ) {
-        this(writeAlias,IndexingUtils.createIndexDocId(applicationScope, entity,indexEdge),
EntityToMapConverter.convert(applicationScope,indexEdge, entity));
+
+
+        this(writeAlias,IndexingUtils.createIndexDocId(applicationScope, entity,indexEdge)
+            ,EntityToMapConverter.convert(applicationScope,indexEdge, entity));
+
 
     }
 
+    public IndexOperation( final String writeAlias, final ApplicationScope applicationScope,
IndexEdge indexEdge,
+                           Id entityId, UUID version, Map<String,Object> data ) {
+
+        this(writeAlias,IndexingUtils.createIndexDocId(applicationScope, entityId,version,indexEdge)
+            ,data);
+    }
+
     public IndexOperation( final String writeAlias, String documentId, Map<String, Object>
data ) {
         this.writeAlias = writeAlias;
         this.data = data;
@@ -107,4 +120,8 @@ public class IndexOperation implements BatchOperation {
         result = 31 * result + data.hashCode();
         return result;
     }
+
+    public Map convertedEntityToBeIndexed(ApplicationScope applicationScope,IndexEdge indexEdge,Entity
entity){
+        return EntityToMapConverter.convert(applicationScope,indexEdge, entity);
+    }
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0480b48/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index 2e4a3ff..ab46681 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -42,7 +42,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.ArrayList;
 
-
 import static org.junit.Assert.*;
 
 
@@ -230,6 +229,106 @@ public class CollectionsResourceIT extends AbstractRestIT {
 
     }
 
+    @Test
+    public void postToCollectionArraySchemaWithSchemaFirst() throws Exception {
+
+        //Include the property labeled two to be index.
+        ArrayList<String> indexingArray = new ArrayList<>(  );
+        indexingArray.add( "two" );
+        indexingArray.add( "one.key" );
+
+        //field "fields" is required.
+        Entity payload = new Entity();
+        payload.put( "fields", indexingArray);
+
+        //Post index to the collection metadata
+        this.app().collection( "testCollection" ).collection( "_indexes" ).post( payload
);
+        refreshIndex();
+
+        Map<String,Object> arrayFieldsForTesting = new HashMap<>();
+
+        arrayFieldsForTesting.put( "key","value" );
+        arrayFieldsForTesting.put( "anotherKey","value2");
+
+        //Create test collection with a test entity that is partially indexed.
+        Entity testEntity = new Entity();
+        testEntity.put( "one", arrayFieldsForTesting );
+        testEntity.put( "two","query" );
+
+        //Post entity.
+        this.app().collection( "testCollection" ).post( testEntity );
+        refreshIndex();
+
+        //Do a query to see if you can find the indexed query.
+        String query = "one.key = 'value'";
+        QueryParameters queryParameters = new QueryParameters().setQuery(query);
+
+        //having a name breaks it. Need to get rid of the stack trace and also
+        Collection tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
+        Entity reindexedEntity = tempEntity.getResponse().getEntity();
+        assertEquals( "value2",((Map)reindexedEntity.get( "one" )).get( "anotherKey" ) );
+
+        //Verify if you can query on an entity that was not indexed and that no entities
are returned.
+        query = "one.anotherKey = 'value2'";
+        queryParameters = new QueryParameters().setQuery(query);
+        tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
+        assertEquals(0,tempEntity.getResponse().getEntities().size());
+
+    }
+
+    //test to reflect if one would index whatever is prefixed with that. Will
+    //need to do extensive testing to ensure that it will work with different instances.
+    @Test
+    public void postToCollectionSchemaArrayWithTopLevelIndexing() throws Exception {
+
+        //Include the property labeled two to be index.
+        ArrayList<String> indexingArray = new ArrayList<>(  );
+        indexingArray.add( "two" );
+        //this should work such that one.key and one.anotherKey should work.
+        indexingArray.add( "one" );
+
+        //field "fields" is required.
+        Entity payload = new Entity();
+        payload.put( "fields", indexingArray);
+
+        //Post index to the collection metadata
+        this.app().collection( "testCollection" ).collection( "_indexes" ).post( payload
);
+        refreshIndex();
+
+        Map<String,Object> arrayFieldsForTesting = new HashMap<>();
+
+        arrayFieldsForTesting.put( "key","value" );
+        arrayFieldsForTesting.put( "anotherKey","value2");
+
+        //Create test collection with a test entity that is partially indexed.
+        Entity testEntity = new Entity();
+        testEntity.put( "one", arrayFieldsForTesting );
+        testEntity.put( "two","query" );
+
+        //Post entity.
+        this.app().collection( "testCollection" ).post( testEntity );
+        refreshIndex();
+
+        //Do a query to see if you can find the indexed query.
+        String query = "one.key = 'value'";
+        QueryParameters queryParameters = new QueryParameters().setQuery(query);
+
+        //having a name breaks it. Need to get rid of the stack trace and also
+        Collection tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
+        Entity reindexedEntity = tempEntity.getResponse().getEntity();
+        assertEquals( "value2",((Map)reindexedEntity.get( "one" )).get( "anotherKey" ) );
+
+        //Verify if you can query on an entity that was not indexed and that no entities
are returned.
+        //TODO: check that the below gets indexed as well. although the above should prove
that at least one thing is getting indexed.
+//        query = "one.anotherKey = 'value2'";
+//        queryParameters = new QueryParameters().setQuery(query);
+//        tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
+//        assertEquals(0,tempEntity.getResponse().getEntities().size());
+
+    }
+
+
+
 
     /**
      * Test posts with a user level token on a path with permissions


Mime
View raw message