usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sfeld...@apache.org
Subject [1/5] incubator-usergrid git commit: latest
Date Wed, 18 Mar 2015 21:59:52 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/USERGRID-480 [created] 89f2c2f1f


latest


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/98bc9a42
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/98bc9a42
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/98bc9a42

Branch: refs/heads/USERGRID-480
Commit: 98bc9a4254f23cc07d437a1e818fa9438c0d04f7
Parents: b53cb07
Author: Shawn Feldman <sfeldman@apache.org>
Authored: Tue Mar 17 21:13:32 2015 -0600
Committer: Shawn Feldman <sfeldman@apache.org>
Committed: Tue Mar 17 21:13:32 2015 -0600

----------------------------------------------------------------------
 .../index/impl/EsEntityIndexBatchImpl.java      |  27 ++---
 .../index/impl/EsEntityIndexImpl.java           |  20 ++--
 .../persistence/index/impl/EsQueryVistor.java   |  88 +++++++++-------
 .../persistence/index/impl/IndexingUtils.java   | 104 +++++++++++--------
 4 files changed, 135 insertions(+), 104 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/98bc9a42/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 92312d2..34e7702 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -56,15 +56,7 @@ import com.codahale.metrics.Timer;
 import rx.Observable;
 import rx.functions.Func1;
 
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT_FIELDNAME;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
 
 
 public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@ -243,7 +235,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
             if ( f instanceof ListField ) {
                 List list = ( List ) field.getValue();
-                entityMap.put( field.getName().toLowerCase(),
+                entityMap.put( ARRAY_PREFIX+ field.getName().toLowerCase(),
                         new ArrayList( processCollectionForMap( list ) ) );
 
                 if ( !list.isEmpty() ) {
@@ -255,7 +247,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
             }
             else if ( f instanceof ArrayField ) {
                 List list = ( List ) field.getValue();
-                entityMap.put( field.getName().toLowerCase(),
+                entityMap.put( ARRAY_PREFIX+ field.getName().toLowerCase(),
                         new ArrayList( processCollectionForMap( list ) ) );
             }
             else if ( f instanceof SetField ) {
@@ -265,7 +257,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
             }
             else if ( f instanceof EntityObjectField ) {
                 EntityObject eo = ( EntityObject ) field.getValue();
-                entityMap.put( field.getName().toLowerCase(), entityToMap( eo ) ); // recursion
+                entityMap.put( field.getName().toLowerCase(), entityToMap(eo) ); // recursion
             }
             else if ( f instanceof StringField ) {
 
@@ -284,12 +276,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
                 locMap.put( "lon", locField.getValue().getLongitude() );
                 entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap );
             }
-            else if (  f instanceof DoubleField
-                    || f instanceof FloatField
-                    || f instanceof IntegerField
-                    || f instanceof LongField ) {
-
-                entityMap.put( NUMBER_PREFIX + field.getName().toLowerCase(), field.getValue()
);
+            else if( f instanceof DoubleField || f instanceof  FloatField){
+                entityMap.put( DOUBLE_PREFIX + field.getName().toLowerCase(), field.getValue()
);
+            }
+            else if( f instanceof LongField || f instanceof IntegerField){
+                entityMap.put( LONG_PREFIX + field.getName().toLowerCase(), field.getValue()
);
             }
             else if ( f instanceof BooleanField ) {
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/98bc9a42/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 16f4b3f..598c8d9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -457,11 +457,19 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
 
                 logger.debug( "   Sort: {} order by {}", stringFieldName, order.toString()
);
 
-                final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
-                final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName
)
+                final String longFieldName = LONG_PREFIX + sp.getPropertyName();
+                final FieldSortBuilder longSort = SortBuilders.fieldSort( longFieldName )
                         .order( order ).ignoreUnmapped( true );
-                srb.addSort( numberSort );
-                logger.debug( "   Sort: {} order by {}", numberFieldName, order.toString()
);
+                srb.addSort( longSort );
+                logger.debug( "   Sort: {} order by {}", longFieldName, order.toString()
);
+
+
+                final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName();
+                final FieldSortBuilder doubleSort = SortBuilders.fieldSort( doubleFieldName
)
+                    .order( order ).ignoreUnmapped( true );
+                srb.addSort( doubleSort );
+                logger.debug( "   Sort: {} order by {}", doubleFieldName, order.toString()
);
+
 
                 final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
                 final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName
)
@@ -613,7 +621,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
             }
         };
 
-        doInRetry( retryOperation );
+        doInRetry(retryOperation);
     }
 
 
@@ -741,7 +749,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
                new ClusterHealthRequest( new String[] { indexIdentifier.getIndex( null )
} ) );
 
             //only wait 2 seconds max
-            ClusterHealthResponse chr = future.actionGet( 2000 );
+            ClusterHealthResponse chr = future.actionGet(2000);
             return Health.valueOf( chr.getStatus().name() );
         }
         catch ( Exception ex ) {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/98bc9a42/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index f012bab..8e8b43c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -50,15 +50,11 @@ import org.apache.usergrid.persistence.index.query.tree.WithinOperand;
 
 import com.google.common.base.Joiner;
 
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
 
 
 /**
- * Visits tree of  parsed Query operands and populates 
+ * Visits tree of  parsed Query operands and populates
  * ElasticSearch QueryBuilder that represents the query.
  */
 public class EsQueryVistor implements QueryVisitor {
@@ -67,7 +63,7 @@ public class EsQueryVistor implements QueryVisitor {
     Stack<QueryBuilder> stack = new Stack<QueryBuilder>();
     List<FilterBuilder> filterBuilders = new ArrayList<FilterBuilder>();
 
-    
+
     @Override
     public void visit( AndOperand op ) throws IndexException {
 
@@ -157,14 +153,14 @@ public class EsQueryVistor implements QueryVisitor {
         Object value = op.getLiteral().getValue();
 
         BoolQueryBuilder qb = QueryBuilders.boolQuery(); // let's do a boolean OR
-        qb.minimumNumberShouldMatch(1); 
+        qb.minimumNumberShouldMatch(1);
 
         // field is an entity/array that needs no name prefix
         qb = qb.should( QueryBuilders.matchQuery( name, value ) );
 
         // OR field is a string and needs the prefix on the name
         qb = qb.should( QueryBuilders.matchQuery( addPrefix( value.toString(), name, true),
value));
-        
+
         stack.push( qb );
     }
 
@@ -186,7 +182,7 @@ public class EsQueryVistor implements QueryVisitor {
         FilterBuilder fb = FilterBuilders.geoDistanceFilter( name )
            .lat( lat ).lon( lon ).distance( distance, DistanceUnit.METERS );
         filterBuilders.add( fb );
-    } 
+    }
 
 
     @Override
@@ -222,19 +218,19 @@ public class EsQueryVistor implements QueryVisitor {
             qb.minimumNumberShouldMatch(1);
 
             // field is an entity/array that does not need a prefix on its name
-            // TODO is this right now that we've updated our doc structure?  
+            // TODO is this right now that we've updated our doc structure?
             // Should this be "must" instead of should?
             qb = qb.should( QueryBuilders.wildcardQuery( name, svalue ) );
-           
+
             // or field is just a string that does need a prefix
             if ( svalue.indexOf("*") != -1 ) {
                 qb = qb.should( QueryBuilders.wildcardQuery( addPrefix( value, name ), svalue
) );
             } else {
                 qb = qb.should( QueryBuilders.termQuery(     addPrefix( value, name ), value
));
-            } 
+            }
             stack.push( qb );
             return;
-        } 
+        }
 
         // assume all other types need prefix
         stack.push( QueryBuilders.termQuery( addPrefix( value, name ), value ));
@@ -276,29 +272,38 @@ public class EsQueryVistor implements QueryVisitor {
         if ( parts.length > 1 ) {
             name = parts[ parts.length - 1 ];
         }
-        
+
         if ( value instanceof String && analyzed ) {
             name = addAnalyzedStringPrefix( name );
 
         } else if ( value instanceof String ) {
             name = addStringPrefix( name );
 
-        } else if ( value instanceof Number ) {
-            name = addNumberPrefix( name );
+        }else if ( value instanceof Integer ) {
+            name = addLongPrefix(name);
+
+        }else if ( value instanceof Long ) {
+            name = addLongPrefix(name);
+
+        } else if ( value instanceof Float ) {
+            name = addDoublePrefix(name);
+
+        }else if ( value instanceof Float ) {
+            name = addDoublePrefix(name);
 
         } else if ( value instanceof Boolean ) {
-            name = addBooleanPrefix( name );
+            name = addBooleanPrefix(name);
 
         } else if ( value instanceof UUID ) {
-            name = addStringPrefix( name );
+            name = addStringPrefix(name);
         }
 
-        // re-create nested property name 
+        // re-create nested property name
         if ( parts.length > 1 ) {
             parts[parts.length - 1] = name;
             Joiner joiner = Joiner.on(".").skipNulls();
             return joiner.join(parts);
-        } 
+        }
 
         return name;
     }
@@ -307,34 +312,41 @@ public class EsQueryVistor implements QueryVisitor {
     private String addAnalyzedStringPrefix( String name ) {
         if ( name.startsWith( ANALYZED_STRING_PREFIX ) ) {
             return name;
-        }  
+        }
         return ANALYZED_STRING_PREFIX + name;
-    } 
-   
+    }
+
 
     private String addStringPrefix( String name ) {
         if ( name.startsWith( STRING_PREFIX ) ) {
             return name;
-        } 
+        }
         return STRING_PREFIX + name;
-    } 
-   
+    }
 
-    private String addNumberPrefix( String name ) {
-        if ( name.startsWith( NUMBER_PREFIX ) ) {
+
+    private String addDoublePrefix( String name ) {
+        if ( name.startsWith( DOUBLE_PREFIX ) ) {
             return name;
-        } 
-        return NUMBER_PREFIX + name;
-    } 
-   
+        }
+        return DOUBLE_PREFIX + name;
+    }
+
+    private String addLongPrefix( String name ) {
+        if ( name.startsWith( LONG_PREFIX ) ) {
+            return name;
+        }
+        return LONG_PREFIX + name;
+    }
+
 
     private String addBooleanPrefix( String name ) {
         if ( name.startsWith( BOOLEAN_PREFIX ) ) {
             return name;
-        } 
+        }
         return BOOLEAN_PREFIX + name;
-    } 
-   
+    }
+
 
     @Override
     public QueryBuilder getQueryBuilder() {
@@ -354,10 +366,10 @@ public class EsQueryVistor implements QueryVisitor {
 			for ( FilterBuilder fb : filterBuilders ) {
 				if ( andFilter == null ) {
 					andFilter = FilterBuilders.andFilter( fb );
-				} else {	
+				} else {
 					andFilter = FilterBuilders.andFilter( andFilter, fb );
 				}
-			}	
+			}
 
 		} else if ( !filterBuilders.isEmpty() ) {
 			return filterBuilders.get(0);

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/98bc9a42/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index ffd98e9..0df3a2c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -35,8 +35,11 @@ public class IndexingUtils {
     public static final String STRING_PREFIX = "su_";
     public static final String ANALYZED_STRING_PREFIX = "sa_";
     public static final String GEO_PREFIX = "go_";
-    public static final String NUMBER_PREFIX = "nu_";
+    public static final String LONG_PREFIX = "long_";
+    public static final String DOUBLE_PREFIX = "long_";
+
     public static final String BOOLEAN_PREFIX = "bu_";
+    public static final String ARRAY_PREFIX = "ar_";
 
     public static final String SPLITTER = "\\__";
 
@@ -54,6 +57,7 @@ public class IndexingUtils {
 
     public static final String ENTITY_VERSION_FIELDNAME = "ug_entityVersion";
 
+    public static final String DOC_VALUES_KEY = "doc_values";
 
     /**
       * Create our sub scope.  This is the ownerUUID + type
@@ -151,78 +155,94 @@ public class IndexingUtils {
 
             .startObject()
 
-                    /**  add routing  "_routing":{ "required":false,  "path":"ug_entityId"
**/
-                     .startObject("_routing").field("required",true).field("path",ENTITYID_ID_FIELDNAME).endObject()
-                     .startArray("dynamic_templates")
+                    /**  add routing  "_routing":{ "required":true,  "path":"ug_entityId"
**/
+                    .startObject("_routing")
+                        .field("required", true)
+                        .field("path", ENTITYID_ID_FIELDNAME)
+                    .endObject()
+                    .startArray("dynamic_templates")
                         // we need most specific mappings first since it's a stop on match
algorithm
-
                         .startObject()
-
-                            .startObject( "entity_id_template" )
-                                .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
-                                    .field( "match_mapping_type", "string" )
-                                            .startObject( "mapping" ).field( "type", "string"
)
-                                                .field( "index", "not_analyzed" )
+                            .startObject("entity_id_template")
+                                .field("match", IndexingUtils.ENTITYID_ID_FIELDNAME)
+                                    .field("match_mapping_type", "string")
+                                            .startObject("mapping").field("type", "string")
+                                                .field("index", "not_analyzed")
                                             .endObject()
                                     .endObject()
                                 .endObject()
 
                             .startObject()
-                            .startObject( "entity_context_template" )
-                                .field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
-                                .field( "match_mapping_type", "string" )
-                                    .startObject( "mapping" )
-                                        .field( "type", "string" )
-                                        .field( "index", "not_analyzed" ).endObject()
-                                    .endObject()
+                                .startObject("entity_context_template")
+                                    .field("match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME)
+                                    .field("match_mapping_type", "string")
+                                        .startObject("mapping")
+                                            .field("type", "string")
+                                            .field("index", "not_analyzed")
+                                        .endObject()
+                                .endObject()
                             .endObject()
 
                             .startObject()
-                            .startObject( "entity_version_template" )
-                                .field( "match", IndexingUtils.ENTITY_VERSION_FIELDNAME )
-                                        .field( "match_mapping_type", "string" )
-                                            .startObject( "mapping" ).field( "type", "long"
)
-                                            .endObject()
-                                        .endObject()
+                                .startObject("entity_version_template")
+                                    .field("match", IndexingUtils.ENTITY_VERSION_FIELDNAME)
+                                    .field("match_mapping_type", "string")
+                                    .startObject("mapping").field("type", "long").field(DOC_VALUES_KEY,
true)
                                     .endObject()
+                                .endObject()
+                            .endObject()
 
                             // any string with field name that starts with sa_ gets analyzed
                             .startObject()
-                                .startObject( "template_1" )
-                                    .field( "match", ANALYZED_STRING_PREFIX + "*" )
-                                    .field( "match_mapping_type", "string" ).startObject(
"mapping" )
-                                    .field( "type", "string" )
-                                    .field( "index", "analyzed" )
+                                .startObject("template_string_analyzed")
+                                    .field("match", ANALYZED_STRING_PREFIX + "*")
+                                    .field("match_mapping_type", "string")
+                                    .startObject("mapping")
+                                        .field("type", "string")
+                                        .field("index", "analyzed")
+                                    .endObject()
                                 .endObject()
                             .endObject()
 
-                        .endObject()
-
                         // all other strings are not analyzed
                         .startObject()
-                            .startObject( "template_2" )
+                            .startObject("template_string_not_analyzed")
                                 //todo, should be string prefix, remove 2 field mapping
-                                .field( "match", "*" )
-                                .field( "match_mapping_type", "string" )
-                                .startObject( "mapping" )
-                                    .field( "type", "string" )
-                                        .field( "index", "not_analyzed" )
+                                .field("match", "*")
+                                .field("match_mapping_type", "string")
+                                .startObject("mapping")
+                                    .field("type", "string")
+                                    .field("index", "not_analyzed")
                                 .endObject()
                             .endObject()
                         .endObject()
 
                         // fields names starting with go_ get geo-indexed
                         .startObject()
-                            .startObject( "template_3" )
-                                .field( "match", GEO_PREFIX + "location" )
-                                    .startObject( "mapping" )
-            .field( "type", "geo_point" )
+                            .startObject("template_geo")
+                                .field("match", GEO_PREFIX + "location")
+                                    .startObject("mapping").field("type", "geo_point")
                                     .endObject()
                             .endObject()
                         .endObject()
+                            // all other strings are not analyzed
+                        .startObject()
+                            .startObject("template__long")
+                                .field("match", LONG_PREFIX + "*")
+                                .field("match_mapping_type", "long")
+                                .startObject("mapping").field("type", "long").field("index","not_analyzed").field(DOC_VALUES_KEY,
true).endObject()
+                            .endObject()
+                        .endObject()
 
-                    .endArray()
+                        .startObject()
+                            .startObject("template__double")
+                                .field("match", DOUBLE_PREFIX + "*")
+                                .field("match_mapping_type", "double")
+                                .startObject("mapping").field("type", "double").field("index",
"not_analyzed").field(DOC_VALUES_KEY, true).endObject()
+                            .endObject()
+                        .endObject()
 
+                    .endArray()
             .endObject();
 
         return builder;


Mime
View raw message