usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject [34/36] usergrid git commit: Add some more comments and clarity to selective indexing.
Date Fri, 08 Apr 2016 20:21:21 GMT
Add some more comments and clarity to selective indexing.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9f3720e9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9f3720e9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9f3720e9

Branch: refs/heads/release-2.1.1
Commit: 9f3720e998edd6343d78ec1b5360f1ce244a4d49
Parents: e5ae2a5
Author: George Reyes <grey@apache.org>
Authored: Mon Apr 4 10:55:44 2016 -0700
Committer: George Reyes <grey@apache.org>
Committed: Mon Apr 4 10:55:44 2016 -0700

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        | 36 ++++++++------------
 .../index/ReIndexServiceImpl.java               |  1 -
 .../core/consistency/TimeServiceImpl.java       |  5 ++-
 .../index/impl/EntityToMapConverter.java        | 23 +++++++------
 4 files changed, 31 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f3720e9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 6bbf0a3..4ee497c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -18,14 +18,11 @@ package org.apache.usergrid.corepersistence;
 
 import java.nio.ByteBuffer;
 import java.time.Instant;
-import java.time.ZoneOffset;
-import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -38,22 +35,15 @@ import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.UUID;
 
-
-import org.apache.usergrid.corepersistence.index.IndexSchemaCache;
-import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
-import org.apache.usergrid.corepersistence.service.CollectionService;
-import org.apache.usergrid.corepersistence.service.ConnectionService;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.utils.*;
-import org.apache.usergrid.utils.ClassUtils;
-import org.apache.usergrid.utils.UUIDUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.Assert;
 
-import org.apache.avro.generic.GenericData;
-
 import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.index.IndexSchemaCache;
+import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
+import org.apache.usergrid.corepersistence.service.CollectionService;
+import org.apache.usergrid.corepersistence.service.ConnectionService;
 import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
 import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.persistence.AggregateCounter;
@@ -97,6 +87,7 @@ import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
 import org.apache.usergrid.persistence.graph.GraphManager;
 import org.apache.usergrid.persistence.graph.GraphManagerFactory;
 import org.apache.usergrid.persistence.graph.SearchEdgeType;
+import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.query.CounterResolution;
 import org.apache.usergrid.persistence.index.query.Identifier;
 import org.apache.usergrid.persistence.map.MapManager;
@@ -106,6 +97,13 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.apache.usergrid.persistence.model.field.Field;
 import org.apache.usergrid.persistence.model.field.StringField;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.utils.ClassUtils;
+import org.apache.usergrid.utils.CompositeUtils;
+import org.apache.usergrid.utils.InflectionUtils;
+import org.apache.usergrid.utils.Inflector;
+import org.apache.usergrid.utils.JsonUtils;
+import org.apache.usergrid.utils.StringUtils;
+import org.apache.usergrid.utils.UUIDUtils;
 
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Timer;
@@ -1751,17 +1749,14 @@ public class CpEntityManager implements EntityManager {
     @Override
     public Map createCollectionSchema( String collectionName, String owner ,Map<String,
Object> properties ){
 
-        //haven't decided which one I should base off of which, maybe from epoch to utc
 
         //TODO: change timeservice as below then use timeservice.
-        //TODO: only allow a single reindex in elasticsearch at a time. akka.
         Instant timeInstance = Instant.now();
 
         Long epoch = timeInstance.toEpochMilli();
 
         Map<String,Object> schemaMap = new HashMap<>(  );
 
-
         schemaMap.put("lastUpdated",epoch);
         //this needs the method that can extract the user from the token no matter the token.
         //Possible values are app credentials, org credentials, or the user email(Admin tokens).
@@ -1775,7 +1770,7 @@ public class CpEntityManager implements EntityManager {
         java.util.Optional<Map> collectionIndexingSchema = indexSchemaCache.getCollectionSchema(
collectionName );
 
 
-        //If we do have a schema then parse it and add it to a list of properties we want
to keep.Otherwise return.
+        //If there is an existing schema then take the lastReindexed time and keep it around.Otherwise
initialize to 0.
         if ( collectionIndexingSchema.isPresent() ) {
             Map jsonMapData = collectionIndexingSchema.get();
             schemaMap.put( "lastReindexed", jsonMapData.get( "lastReindexed" ) );
@@ -1786,6 +1781,8 @@ public class CpEntityManager implements EntityManager {
 
         ArrayList<String> fieldProperties = ( ArrayList<String> ) properties.get(
"fields" );
 
+        //do a check to see if you have a * field. If you do have a * field then ignore all
other fields
+        //and only accept the * field.
         if(fieldProperties.contains( "*" )){
             ArrayList<String> wildCardArrayList = new ArrayList<>(  );
             wildCardArrayList.add( "*" );
@@ -1795,9 +1792,6 @@ public class CpEntityManager implements EntityManager {
             schemaMap.putAll( properties );
         }
 
-        //do a check to see if you have a * field. If you do have a * field then ignore all
other fields
-        //and only accept the * field. That logic goes below and in the put.
-
         indexSchemaCache.putCollectionSchema( collectionName, JsonUtils.mapToJsonString(
schemaMap ) );
 
         return schemaMap;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f3720e9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index 125222a..0aa0f60 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -44,7 +44,6 @@ import org.apache.usergrid.persistence.map.MapManager;
 import org.apache.usergrid.persistence.map.MapManagerFactory;
 import org.apache.usergrid.persistence.map.MapScope;
 import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 import org.apache.usergrid.utils.InflectionUtils;
 import org.apache.usergrid.utils.JsonUtils;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f3720e9/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java
b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java
index 00a58d3..99fe756 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java
@@ -20,6 +20,9 @@
 package org.apache.usergrid.persistence.core.consistency;
 
 
+import java.time.Instant;
+
+
 /**
  * Simple time service to get the current system time.
  */
@@ -28,6 +31,6 @@ public class TimeServiceImpl implements TimeService{
 
     @Override
     public long getCurrentTime() {
-        return System.currentTimeMillis();
+        return Instant.now().toEpochMilli();
     }
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f3720e9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
index 98a8ed5..8eb9f48 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
@@ -65,6 +65,7 @@ public class EntityToMapConverter {
      * @param applicationScope
      * @param entity The entity
      * @param indexEdge The edge this entity is indexed on
+     * @param fieldsToIndex A set of fields that will be indexed should they exist on the
entity. Other fields will be filtered out.
      */
     public static Map<String, Object> convert(ApplicationScope applicationScope, final
IndexEdge indexEdge,
                                               final Entity entity, Optional<Set<String>>
fieldsToIndex) {
@@ -114,7 +115,6 @@ public class EntityToMapConverter {
 
         if(fieldsToIndex.isPresent()){
             Set<String> defaultProperties = fieldsToIndex.get();
-            //copy paste logic here.
             HashSet mapFields = ( HashSet ) outputEntity.get( "fields" );
             Iterator collectionIterator = mapFields.iterator();
 
@@ -137,10 +137,11 @@ public class EntityToMapConverter {
     }
 
     /**
-     * This method is crucial for selective top level indexing. Here we check to see if the
flatted properties
-     * are in fact a top level exclusion e.g one.two.three and one.three.two can be allowed
for querying by
-     * specifying one in the schema. If they are not a top level exclusion then they are
removed from the iterator and
-     * the map.
+     * Handles checking to see if a field is a top level exclusion or just a field that shouldn't
be indexed.
+     * This is handled by looping through all the fields we want to be able to query on,
and checking to see if a
+     * specific field name is included. If the field name is included then do nothing. If
the field name is not included
+     * then we do not want to be able to query on it. Instead we remove it from the collectionIterator
which
+     * removes it from the outputEntity above, thus filtering it out.
      *
      * @param fieldsToKeep - contains a list of fields that the user defined in their schema.
      * @param collectionIterator - contains the iterator with the reference to the map where
we want to remove the field. Once removed here it is removed from the entity so it won't be
indexed.
@@ -154,12 +155,12 @@ public class EntityToMapConverter {
 
 
 
-        //goes through a loop of all the fields ( excluding default ) that we want to keep.
-        //if the toRemoveFlag is set to false then we want to keep the property, otherwise
we set it to true and remove
+        //goes through a loop of all the fields that we want to keep.
+        //if the toRemoveFlag is set to false then we want to keep the property and do nothing
to it, otherwise we set it to true and remove
         //the property.
         while ( fieldIterator.hasNext() ) {
             //this is the field that we're
-            String requiredInclusionString = ( String ) fieldIterator.next();
+            String fieldToKeep = ( String ) fieldIterator.next();
 
 
             //Since we know that the fieldName cannot be equal to the requiredInclusion criteria
due to the if condition before we enter this method
@@ -169,9 +170,9 @@ public class EntityToMapConverter {
             //The second part of the if loop also requires that the fieldName is followed
by a period after we check to ensure that the
             //indexing criteria is included in the string. This is done to weed out values
such as one.twoexample.three
             // when we should only keep one.two.three when comparing the indexing criteria
of one.two.
-            if(fieldName.length() > requiredInclusionString.length()
-                && fieldName.contains( requiredInclusionString )
-                && fieldName.charAt( requiredInclusionString.length() )=='.' ) {
+            if(fieldName.length() > fieldToKeep.length()
+                && fieldName.contains( fieldToKeep )
+                && fieldName.charAt( fieldToKeep.length() )=='.' ) {
                 toRemoveFlag = false;
                 break;
             }


Mime
View raw message