usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject [17/36] usergrid git commit: Working lower level selective indexing. Working field updates when we do reindexing and stuff. Added test proving that it works.
Date Fri, 08 Apr 2016 20:21:04 GMT
Working lower level selective indexing.
Working field updates when we do reindexing and stuff.
Added test proving that it works.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b5d81ac0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b5d81ac0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b5d81ac0

Branch: refs/heads/release-2.1.1
Commit: b5d81ac0334fc5aeb204f137bfcbd500c7c1086c
Parents: 82ff53b
Author: George Reyes <grey@apache.org>
Authored: Tue Mar 29 14:11:56 2016 -0700
Committer: George Reyes <grey@apache.org>
Committed: Tue Mar 29 14:11:56 2016 -0700

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        |  18 ++-
 .../corepersistence/index/IndexServiceImpl.java |  52 ++------
 .../index/ReIndexServiceImpl.java               |  26 +++-
 .../persistence/index/EntityIndexBatch.java     |  13 +-
 .../index/impl/EntityToMapConverter.java        | 133 ++++++++++++++++++-
 .../index/impl/EsEntityIndexBatchImpl.java      |  27 ++--
 .../persistence/index/impl/IndexOperation.java  |  20 +--
 .../persistence/index/impl/EntityIndexTest.java |   4 +-
 .../collection/CollectionsResourceIT.java       |  25 +++-
 .../test/resource/endpoints/NamedResource.java  |  22 +++
 10 files changed, 242 insertions(+), 98 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index bb3abcb..2c97bc6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -1737,7 +1737,6 @@ public class CpEntityManager implements EntityManager {
     @Override
     public Map createCollectionSchema( String collectionName, String owner ,Map<String,
Object> properties ){
 
-
         //haven't decided which one I should base off of which, maybe from epoch to utc
 
         //TODO: change timeservice as below then use timeservice.
@@ -1753,10 +1752,23 @@ public class CpEntityManager implements EntityManager {
         //this needs the method that can extract the user from the token no matter the token.
         //Possible values are app credentials, org credentials, or the user email(Admin tokens).
         schemaMap.put("lastUpdateBy",owner);
-        schemaMap.put("lastReindexed",0);
-        schemaMap.putAll( properties );
 
+
+        //TODO: please add a check to get the previous reindex time.
         MapManager mm = getMapManagerForTypes();
+
+        String jsonSchemaMap = mm.getString( collectionName );
+
+        //If we do have a schema then parse it and add it to a list of properties we want
to keep.Otherwise return.
+        if ( jsonSchemaMap != null ) {
+            Map jsonMapData = ( Map ) JsonUtils.parse( jsonSchemaMap );
+            schemaMap.put( "lastReindexed", jsonMapData.get( "lastReindexed" ) );
+        }
+        else {
+            schemaMap.put( "lastReindexed", 0 );
+        }
+        schemaMap.putAll( properties );
+
         mm.putString( collectionName,JsonUtils.mapToJsonString( schemaMap ) );
 
         return schemaMap;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 5f50a32..c3d5361 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
@@ -134,14 +135,9 @@ public class IndexServiceImpl implements IndexService {
                         logger.debug("adding edge {} to batch for entity {}", indexEdge,
entity);
                     }
 
-                    final Map map = getFilteredStringObjectMap( applicationScope, entity,
indexEdge );
+                    final Optional<Set<String>> fieldsToIndex = getFilteredStringObjectMap(
applicationScope, entity, indexEdge );
 
-                    if(map!=null){
-                        batch.index( indexEdge, entity ,map);
-                    }
-                    else{
-                        batch.index( indexEdge,entity );
-                    }
+                    batch.index( indexEdge, entity ,fieldsToIndex);
                 } )
                     //return the future from the batch execution
                 .map( batch -> batch.build() ) );
@@ -172,14 +168,9 @@ public class IndexServiceImpl implements IndexService {
                 logger.debug("adding edge {} to batch for entity {}", indexEdge, entity);
             }
 
-            Map map = getFilteredStringObjectMap( applicationScope, entity, indexEdge );
+            Optional<Set<String>> fieldsToIndex = getFilteredStringObjectMap(
applicationScope, entity, indexEdge );
 
-            if(map!=null){
-                batch.index( indexEdge, entity ,map);
-            }
-            else{
-                batch.index( indexEdge,entity );
-            }
+            batch.index( indexEdge, entity ,fieldsToIndex);
 
 
             return batch.build();
@@ -202,9 +193,11 @@ public class IndexServiceImpl implements IndexService {
      * @return This returns a filtered map that contains the flatted properties of the entity.
If there isn't a schema
      * associated with the collection then return null ( and index the entity in its entirety
)
      */
-    private Map getFilteredStringObjectMap( final ApplicationScope applicationScope,
+    private Optional<Set<String>> getFilteredStringObjectMap( final ApplicationScope
applicationScope,
                                             final Entity entity, final IndexEdge indexEdge
) {
 
+
+        //TODO: THIS IS THE FIRST THING TO BE LOOKED AT TOMORROW.
         //look into this.
         IndexOperation indexOperation = new IndexOperation();
 
@@ -233,35 +226,10 @@ public class IndexServiceImpl implements IndexService {
             defaultProperties.addAll( fieldsToKeep );
         }
         else {
-            return null;
-        }
-
-        //Returns the flattened map of the entity.
-        //TODO: maybe instead pass the fields to keep to the flattening.
-        Map map = indexOperation.convertedEntityToBeIndexed( applicationScope, indexEdge,
entity );
-
-        HashSet mapFields = ( HashSet ) map.get( "fields" );
-        Iterator collectionIterator = mapFields.iterator();
-
-        //Loop through all of the fields of the flatted entity and check to see if they should
be filtered out.
-        while ( collectionIterator.hasNext() ) {
-            EntityField testedField = ( EntityField ) collectionIterator.next();
-            String fieldName = ( String ) ( testedField ).get( "name" );
-
-            //Checks to see if the fieldname is a default property. If it is then keep it,
otherwise send it to
-            //be verified the aptly named method
-
-            //one.two.three
-            //one.two.four
-            //one.two3.five
-            //one.two
-            //fields { one.two }
-            if ( !defaultProperties.contains( fieldName ) ) {
-                iterateThroughMapForFieldsToBeIndexed( fieldsToKeep, collectionIterator,
fieldName );
-            }
+            return Optional.empty();
         }
 
-        return map;
+        return Optional.of(defaultProperties);
     }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index 558a18c..42067f5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -20,7 +20,9 @@
 package org.apache.usergrid.corepersistence.index;
 
 
+import java.time.Instant;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 
@@ -42,7 +44,10 @@ import org.apache.usergrid.persistence.map.MapManager;
 import org.apache.usergrid.persistence.map.MapManagerFactory;
 import org.apache.usergrid.persistence.map.MapScope;
 import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.utils.InflectionUtils;
+import org.apache.usergrid.utils.JsonUtils;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Optional;
@@ -76,6 +81,7 @@ public class ReIndexServiceImpl implements ReIndexService {
     private final AllEntityIdsObservable allEntityIdsObservable;
     private final IndexProcessorFig indexProcessorFig;
     private final MapManager mapManager;
+    private final MapManagerFactory mapManagerFactory;
     private final AsyncEventService indexService;
     private final EntityIndexFactory entityIndexFactory;
 
@@ -94,7 +100,7 @@ public class ReIndexServiceImpl implements ReIndexService {
         this.allApplicationsObservable = allApplicationsObservable;
         this.indexProcessorFig = indexProcessorFig;
         this.indexService = indexService;
-
+        this.mapManagerFactory = mapManagerFactory;
         this.mapManager = mapManagerFactory.createMapManager( RESUME_MAP_SCOPE );
     }
 
@@ -133,12 +139,30 @@ public class ReIndexServiceImpl implements ReIndexService {
             .buffer( indexProcessorFig.getReindexBufferSize());
 
         if(delayTimer.isPresent()){
+
+            if(reIndexRequestBuilder.getCollectionName().isPresent()) {
+                String collectionName =  InflectionUtils.pluralize( CpNamingUtils.getNameFromEdgeType(reIndexRequestBuilder.getCollectionName().get()
));
+                MapManager collectionMapStorage = mapManagerFactory.createMapManager( CpNamingUtils.getEntityTypeMapScope(
appId.get().getApplication()  ) );
+                String jsonSchemaMap = collectionMapStorage.getString( collectionName );
+
+
+                //If we do have a schema then parse it and add it to a list of properties
we want to keep.Otherwise return.
+                if ( jsonSchemaMap != null ) {
+
+                    Map jsonMapData = ( Map ) JsonUtils.parse( jsonSchemaMap );
+
+                    jsonMapData.put( "lastReindexed", Instant.now().toEpochMilli() );
+                    collectionMapStorage.putString( collectionName,JsonUtils.mapToJsonString(jsonMapData
)  );
+                }
+
+            }
             if(timeUnitOptional.isPresent()){
                 runningReIndex = runningReIndex.delay( delayTimer.get(),timeUnitOptional.get()
);
             }
             else{
                 runningReIndex = runningReIndex.delay( delayTimer.get(), TimeUnit.MILLISECONDS
);
             }
+
         }
 
         runningReIndex = runningReIndex.doOnNext(edges -> {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
index e3a40f6..2e5ece6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
@@ -20,6 +20,8 @@ package org.apache.usergrid.persistence.index;/*
 
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import java.util.UUID;
 
 import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
@@ -30,13 +32,7 @@ import rx.Observable;
 
 public interface EntityIndexBatch {
 
-    /**
-     * Create index for Entity
-     *
-     * @param indexEdge  The edge to index the document into
-     * @param entity     Entity to be indexed.
-     */
-    EntityIndexBatch index( final IndexEdge indexEdge, final Entity entity );
+    EntityIndexBatch index( IndexEdge indexEdge, Entity entity );
 
     /**
      * Remove index of entity
@@ -54,7 +50,8 @@ public interface EntityIndexBatch {
      */
     EntityIndexBatch deindex( final SearchEdge searchEdge, final CandidateResult result );
 
-    EntityIndexBatch index( IndexEdge indexEdge, Entity entity, Map flattenedEntityMap );
+
+    EntityIndexBatch index( IndexEdge indexEdge, Entity entity, Optional<Set<String>>
fieldsToIndex );
 
     /**
      * Remove index of entity.

http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
index b43f4bb..5a728a3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
@@ -17,8 +17,12 @@
 package org.apache.usergrid.persistence.index.impl;
 
 
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -48,6 +52,14 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.nodeId;
  * Convert a CP entity to an elasticsearch document
  */
 public class EntityToMapConverter {
+
+
+    public static Map<String, Object> convert(ApplicationScope applicationScope, final
IndexEdge indexEdge,
+                                              final Entity entity) {
+
+        return convert( applicationScope, indexEdge, entity, Optional.empty() );
+    }
+
     /**
      * Set the entity as a map with the context
      *
@@ -55,7 +67,8 @@ public class EntityToMapConverter {
      * @param entity The entity
      * @param indexEdge The edge this entity is indexed on
      */
-    public static Map<String, Object> convert(ApplicationScope applicationScope, final
IndexEdge indexEdge, final Entity entity) {
+    public static Map<String, Object> convert(ApplicationScope applicationScope, final
IndexEdge indexEdge,
+                                              final Entity entity, Optional<Set<String>>
fieldsToIndex) {
 
 
 
@@ -95,14 +108,128 @@ public class EntityToMapConverter {
         //now visit our entity
         final FieldParser parser = new EntityMappingParser();
 
-        final Set<EntityField> fieldsToIndex =   parser.parse( entityMap );
+        //TODO: to the parsing in the parse method below, that way we don't iterate twice
and we filter things out as we see them.
+        final Set<EntityField> fieldsToBeFiltered =   parser.parse( entityMap );
 
 
         //add our fields
-        outputEntity.put( ENTITY_FIELDS, fieldsToIndex );
+        outputEntity.put( ENTITY_FIELDS, fieldsToBeFiltered );
+
+        if(fieldsToIndex.isPresent()){
+            Set<String> defaultProperties = fieldsToIndex.get();
+            //copy paste logic here.
+            HashSet mapFields = ( HashSet ) outputEntity.get( "fields" );
+            Iterator collectionIterator = mapFields.iterator();
+
+            //Loop through all of the fields of the flatted entity and check to see if they
should be filtered out.
+            while ( collectionIterator.hasNext() ) {
+                EntityField testedField = ( EntityField ) collectionIterator.next();
+                String fieldName = ( String ) ( testedField ).get( "name" );
+
+                //Checks to see if the fieldname is a default property. If it is then keep
it, otherwise send it to
+                //be verified the aptly named method
+
+                //one.two.three
+                //one.two.four
+                //one.two3.five
+                //one.two
+                //fields { one.two }
+                if ( !defaultProperties.contains( fieldName ) ) {
+                    iterateThroughMapForFieldsToBeIndexed( defaultProperties, collectionIterator,
fieldName );
+                }
+            }
+        }
+
 
 
         return outputEntity;
     }
 
+    /**
+     * This method is crucial for selective top level indexing. Here we check to see if the
flatted properties
+     * are in fact a top level exclusion e.g one.two.three and one.three.two can be allowed
for querying by
+     * specifying one in the schema. If they are not a top level exclusion then they are
removed from the iterator and
+     * the map.
+     *
+     * @param fieldsToKeep - contains a list of fields that the user defined in their schema.
+     * @param collectionIterator - contains the iterator with the reference to the map where
we want to remove the field.
+     * @param fieldName - contains the name of the field that we want to keep.
+     */
+    private static void iterateThroughMapForFieldsToBeIndexed( final Set<String> fieldsToKeep,
+                                                               final Iterator collectionIterator,
+                                                               final String fieldName ) {
+        boolean toRemoveFlag = true;
+        String[] flattedStringArray = getStrings( fieldName );
+
+        Iterator fieldIterator = fieldsToKeep.iterator(); //fieldsToKeep.iterator();
+
+        //goes through a loop of all the fields ( excluding default ) that we want to keep.
+        //if the toRemoveFlag is set to false then we want to keep the property, otherwise
we set it to true and remove
+        //the property.
+        while ( fieldIterator.hasNext() ) {
+            String requiredInclusionString = ( String ) fieldIterator.next();
+            String[] flattedRequirementString = getStrings( requiredInclusionString );
+
+
+            //loop each split array value to see if it matches the equivalent value
+            //in the field. e.g in the example one.two.three and one.two.four we need to
check that the schema
+            //matches in both one and two above. If instead it says to exclude one.twor then
we would still exclude the above
+            //since it is required to be a hard match.
+
+            //The way the below works if we see that the current field isn't as fine grained
as the schema rule
+            //( aka the array is shorter than the current index of the schema rule then there
is no way the rule could apply
+            // to the index.
+
+            //Then if that check passes we go to check that both parts are equal. If they
are ever not equal
+            // e.g one.two.three and one.three.two then it shouldn't be included
+            //TODO: regex.
+            for ( int index = 0; index < flattedRequirementString.length; index++ ) {
+                //if the array contains a string that it is equals to then set the remove
flag to true
+                //otherwise remain false.
+
+                //one.three
+                //one.two
+                //one
+                if ( flattedStringArray.length <= index ) {
+                    toRemoveFlag = true;
+                    break;
+                }
+
+                if ( flattedRequirementString[index].equals( flattedStringArray[index] )
) {
+                    toRemoveFlag = false;
+                }
+                else {
+                    toRemoveFlag = true;
+                    break;
+                }
+            }
+            if ( toRemoveFlag == false ) {
+                break;
+            }
+        }
+
+        if ( toRemoveFlag ) {
+            collectionIterator.remove();
+        }
+    }
+
+
+    /**
+     * Splits the string on the flattened period "." seperated values.
+     * @param fieldName
+     * @return
+     */
+    private static String[] getStrings( final String fieldName ) {
+        final String[] flattedStringArray;
+        if ( !fieldName.contains( "." ) ) {
+            //create a single array that is made of a the single value.
+            flattedStringArray = new String[] { fieldName };
+        }
+        else {
+            flattedStringArray = fieldName.split( "\\." );
+        }
+        return flattedStringArray;
+    }
+
 }
+

http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 5296978..ed00f48 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -20,12 +20,16 @@ package org.apache.usergrid.persistence.index.impl;
 
 
 import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import java.util.UUID;
 
 import org.apache.usergrid.persistence.index.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.auth.IAuthenticator;
+
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.util.ValidationUtils;
 import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
@@ -65,24 +69,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
     @Override
     public EntityIndexBatch index( final IndexEdge indexEdge, final Entity entity ) {
-        IndexValidationUtils.validateIndexEdge(indexEdge);
-        ValidationUtils.verifyEntityWrite(entity);
-        ValidationUtils.verifyVersion( entity.getVersion() );
-
-        final String writeAlias = alias.getWriteAlias();
-
-        if ( logger.isDebugEnabled() ) {
-            logger.debug( "Indexing to alias {} with scope {} on edge {} with entity data
{}",
-                    writeAlias, applicationScope, indexEdge, entity.getFieldMap().keySet()
);
-        }
-
-        //add app id for indexing
-        container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, indexEdge,
entity));
-        return this;
+        return index( indexEdge,entity, Optional.empty() );
     }
 
     @Override
-    public EntityIndexBatch index ( final IndexEdge indexEdge, final Entity entity ,final
Map flattenedEntityMap){
+    public EntityIndexBatch index( final IndexEdge indexEdge, final Entity entity, final
Optional<Set<String>> fieldsToIndex ) {
         IndexValidationUtils.validateIndexEdge(indexEdge);
         ValidationUtils.verifyEntityWrite(entity);
         ValidationUtils.verifyVersion( entity.getVersion() );
@@ -91,16 +82,14 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         if ( logger.isDebugEnabled() ) {
             logger.debug( "Indexing to alias {} with scope {} on edge {} with entity data
{}",
-                writeAlias, applicationScope, indexEdge, flattenedEntityMap );
+                    writeAlias, applicationScope, indexEdge, entity.getFieldMap().keySet()
);
         }
 
         //add app id for indexing
-        container.addIndexRequest(new IndexOperation(writeAlias, applicationScope,
-            indexEdge,entity.getId(),entity.getVersion(),flattenedEntityMap) );
+        container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, indexEdge,
entity,fieldsToIndex));
         return this;
     }
 
-
     @Override
     public EntityIndexBatch deindex( final SearchEdge searchEdge, final Id id, final UUID
version ) {
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
index b51d0c7..8f925ed 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
@@ -21,6 +21,8 @@ package org.apache.usergrid.persistence.index.impl;
 
 
 import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 import java.util.UUID;
 
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -49,21 +51,13 @@ public class IndexOperation implements BatchOperation {
     public Map<String, Object> data;
 
     public IndexOperation( final String writeAlias, final ApplicationScope applicationScope,
IndexEdge indexEdge,
-                           Entity entity ) {
-
-
-        this(writeAlias,IndexingUtils.createIndexDocId(applicationScope, entity,indexEdge)
-            ,EntityToMapConverter.convert(applicationScope,indexEdge, entity));
+                           Entity entity, Optional<Set<String>> fieldsToIndex
) {
 
+        this( writeAlias, IndexingUtils.createIndexDocId( applicationScope, entity, indexEdge
),
+            EntityToMapConverter.convert( applicationScope, indexEdge, entity, fieldsToIndex
) );
 
     }
 
-    public IndexOperation( final String writeAlias, final ApplicationScope applicationScope,
IndexEdge indexEdge,
-                           Id entityId, UUID version, Map<String,Object> data ) {
-
-        this(writeAlias,IndexingUtils.createIndexDocId(applicationScope, entityId,version,indexEdge)
-            ,data);
-    }
 
     public IndexOperation( final String writeAlias, String documentId, Map<String, Object>
data ) {
         this.writeAlias = writeAlias;
@@ -120,8 +114,4 @@ public class IndexOperation implements BatchOperation {
         result = 31 * result + data.hashCode();
         return result;
     }
-
-    public Map convertedEntityToBeIndexed(ApplicationScope applicationScope,IndexEdge indexEdge,Entity
entity){
-        return EntityToMapConverter.convert(applicationScope,indexEdge, entity);
-    }
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 3978956..27f3afd 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -365,7 +365,7 @@ public class EntityIndexTest extends BaseIT {
         EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID() );
         entity.setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID()
) );
 
-        indexProducer.put(entityIndex.createBatch().index( searchEdge, entity ).build()).subscribe();
+        indexProducer.put(entityIndex.createBatch().index( searchEdge, entity, null ).build()).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
 
         CandidateResults candidateResults = entityIndex
@@ -412,7 +412,7 @@ public class EntityIndexTest extends BaseIT {
             IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", SearchEdge.NodeType.SOURCE,
System.currentTimeMillis()*1000 );
 
             //index the new entity. This is where the loop will be set to create like 100
entities.
-            indexProducer.put(entityIndex.createBatch().index( searchEdge, entity[i]  ).build()).subscribe();
+            indexProducer.put(entityIndex.createBatch().index( searchEdge, entity[i], null
 ).build()).subscribe();
 
         }
         entityIndex.refreshAsync().toBlocking().first();

http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index 109bb6f..7926e49 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -28,6 +28,7 @@ import org.apache.usergrid.rest.test.resource.model.Collection;
 import org.apache.usergrid.rest.test.resource.model.Entity;
 import org.apache.usergrid.rest.test.resource.model.QueryParameters;
 import org.apache.usergrid.rest.test.resource.model.Token;
+import org.apache.usergrid.services.ServiceParameter;
 
 import org.junit.Ignore;
 import org.junit.Test;
@@ -167,7 +168,7 @@ public class CollectionsResourceIT extends AbstractRestIT {
 
 
         //Reindex and verify that the entity only has field one index.
-        this.app().collection( "testCollection" ).collection( "_reindex" ).post();
+        this.app().collection( "testCollection" ).collection( "_reindex" ).post(true,clientSetup.getSuperuserToken(),ApiResponse.class,null,null,false);
        // Thread.sleep( 10000 );
         //refreshIndex();
 
@@ -225,7 +226,7 @@ public class CollectionsResourceIT extends AbstractRestIT {
         //TODO: the below will have to be replaced by the values that I deem correct.
         assertEquals( ( thing ).get( "lastUpdated" ), testCollectionSchema.get( "lastUpdated"
));
         assertEquals( ( thing ).get( "lastUpdateBy" ),testCollectionSchema.get( "lastUpdateBy"
) );
-        assertEquals( ( thing ).get( "lastReindexed" ),testCollectionSchema.get( "lastReindexed"
) );
+        assertEquals( 0,testCollectionSchema.get( "lastReindexed" ) );
 
         //TODO: this test doesn't check to see if create checks the schema. Only that the
reindex removes whats already there.
         ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get(
"fields" );
@@ -233,14 +234,28 @@ public class CollectionsResourceIT extends AbstractRestIT {
 
 
         //Reindex and verify that the entity only has field one index.
-        this.app().collection( "testCollection" ).collection( "_reindex" ).post();
+        this.app().collection( "testCollection" ).collection( "_reindex" ).post(true,clientSetup.getSuperuserToken(),ApiResponse.class,null,null,false);
+
+
+        indexingArray.add( "one" );
+        indexingArray.add( "two" );
+
+
+        //field "fields" is required.
+        payload = new Entity();
+        payload.put( "fields", indexingArray);
+
+        //Post index to the collection metadata
+        this.app().collection( "testCollection" ).collection( "_indexes" ).post( payload
);
 
         collection = this.app().collection( "testCollection" ).collection( "_index" ).get();
 
+
+
         testCollectionSchema = (LinkedHashMap)collection.getResponse().getData();
-        assertEquals( ( thing ).get( "lastUpdated" ), testCollectionSchema.get( "lastUpdated"
));
+        assertNotEquals( ( thing ).get( "lastUpdated" ), testCollectionSchema.get( "lastUpdated"
));
         assertEquals( ( thing ).get( "lastUpdateBy" ),testCollectionSchema.get( "lastUpdateBy"
) );
-        assertNotEquals( ( thing ).get( "lastReindexed" ),testCollectionSchema.get( "lastReindexed"
) );
+        assertNotEquals( 0,testCollectionSchema.get( "lastReindexed" ) );
 
         schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" );
         assertEquals( "one",schema.get( 0 ) );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5d81ac0/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/NamedResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/NamedResource.java
b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/NamedResource.java
index 0254d18..a2377d4 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/NamedResource.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/NamedResource.java
@@ -272,6 +272,28 @@ public class NamedResource implements UrlResource {
             .post(javax.ws.rs.client.Entity.json(entity), gt);
     }
 
+    // Used for empty posts
+    public <T> T post(boolean useToken, Token tokenToUse , Class<T> type, Map
entity,
+                      final QueryParameters queryParameters, boolean useBasicAuthentication)
{
+
+        WebTarget resource = getTarget( useToken,tokenToUse );
+        resource = addParametersToResource(resource, queryParameters);
+
+        GenericType<T> gt = new GenericType<>((Class) type);
+
+        if (useBasicAuthentication) {
+            HttpAuthenticationFeature feature = HttpAuthenticationFeature.basicBuilder()
+                                                                         .credentials("superuser",
"superpassword").build();
+            return resource.register(feature).request()
+                           .accept(MediaType.APPLICATION_JSON)
+                           .post(javax.ws.rs.client.Entity.json(entity), gt);
+        }
+
+        return resource.request()
+                       .accept(MediaType.APPLICATION_JSON)
+                       .post(javax.ws.rs.client.Entity.json(entity), gt);
+    }
+
     //For edge cases like Organizations and Tokens without any payload
     public <T> T get(Class<T> type) {
         return get(type, null, true);


Mime
View raw message