usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject incubator-usergrid git commit: [USERGRID-460] Added ability to make geo sorts possible. Not Working. Refacorted the query code for filterBuilders and queryBuilders in the Query class.
Date Tue, 17 Mar 2015 23:28:56 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/USERGRID-460 65ff858db -> 65c5e22d1


[USERGRID-460] Added ability to make geo sorts possible. Not Working.
Refacorted the query code for filterBuilders and queryBuilders in the Query class.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/65c5e22d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/65c5e22d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/65c5e22d

Branch: refs/heads/USERGRID-460
Commit: 65c5e22d18d7eaba6e41ff7d84eb51365da6930d
Parents: 65ff858
Author: GERey <greyes@apigee.com>
Authored: Tue Mar 17 16:28:55 2015 -0700
Committer: GERey <greyes@apigee.com>
Committed: Tue Mar 17 16:28:55 2015 -0700

----------------------------------------------------------------------
 .../cassandra/QueryProcessorImpl.java           | 15 +++-
 .../index/impl/EsEntityIndexImpl.java           | 77 +++++++++++++-------
 .../persistence/index/impl/EsQueryVistor.java   | 61 +++++++++-------
 .../usergrid/persistence/index/query/Query.java | 36 ++-------
 .../index/query/tree/QueryVisitor.java          |  5 +-
 5 files changed, 107 insertions(+), 87 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/65c5e22d/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
index 874ff88..7c4ea37 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
@@ -71,6 +71,7 @@ import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
 import org.apache.usergrid.persistence.schema.CollectionInfo;
 import org.elasticsearch.index.query.FilterBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -292,7 +293,7 @@ public class QueryProcessorImpl implements QueryProcessor {
             }
         }
         if (logger.isDebugEnabled()) {
-        	logger.debug("Getting result for query: [{}],  returning entityIds size: {}", 
+        	logger.debug("Getting result for query: [{}],  returning entityIds size: {}",
                     getQuery(), entityIds.size());
         }
 
@@ -621,12 +622,18 @@ public class QueryProcessorImpl implements QueryProcessor {
 
         @Override
         public QueryBuilder getQueryBuilder() {
-            throw new UnsupportedOperationException("Not supported by this vistor implementation.");

+            throw new UnsupportedOperationException("Not supported by this vistor implementation.");
         }
 
         @Override
         public FilterBuilder getFilterBuilder() {
-            throw new UnsupportedOperationException("Not supported by this vistor implementation.");

+            throw new UnsupportedOperationException("Not supported by this vistor implementation.");
+        }
+
+
+        @Override
+        public GeoDistanceSortBuilder getGeoDistanceSortBuilder() {
+            throw new UnsupportedOperationException("Not supported by this vistor implementation.");
         }
     }
 
@@ -724,4 +731,4 @@ public class QueryProcessorImpl implements QueryProcessor {
     public EntityManager getEntityManager() {
         return em;
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/65c5e22d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index c92b299..ed1f03a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -39,6 +39,7 @@ import org.apache.usergrid.persistence.index.exceptions.IndexException;
 import org.apache.usergrid.persistence.index.query.CandidateResult;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
 import org.apache.usergrid.persistence.index.utils.UUIDUtils;
 import org.apache.usergrid.persistence.map.MapManager;
 import org.apache.usergrid.persistence.map.MapManagerFactory;
@@ -64,11 +65,14 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequestBuilder;
 import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.*;
+import org.elasticsearch.index.search.geo.GeoDistanceFilter;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndexMissingException;
 import org.elasticsearch.indices.InvalidAliasNameException;
@@ -397,6 +401,14 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
     }
 
 
+    /**
+     * Needs REfactor to make clearer what queries and how the information gets used so we
don't need to retraverse
+     * the query every time we generate some information about it.
+     * @param indexScope
+     * @param searchTypes
+     * @param query
+     * @return
+     */
     @Override
     public CandidateResults search( final IndexScope indexScope, final SearchTypes searchTypes,
             final Query query ) {
@@ -404,8 +416,9 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
         final String context = IndexingUtils.createContextName(indexScope);
         final String[] entityTypes = searchTypes.getTypeNames();
 
-        QueryBuilder qb = query.createQueryBuilder(context);
+        final QueryVisitor queryVisitor = query.getQueryVisitor();
 
+        QueryBuilder qb = query.createQueryBuilder( context );
 
         SearchResponse searchResponse;
 
@@ -415,8 +428,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
                     .setScroll(cursorTimeout + "m")
                     .setQuery(qb);
 
-            final FilterBuilder fb = query.createFilterBuilder();
-
+            final FilterBuilder fb = queryVisitor.getFilterBuilder();
             //we have post filters, apply them
             if ( fb != null ) {
                 logger.debug( "   Filter: {} ", fb.toString() );
@@ -426,6 +438,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
 
             srb = srb.setFrom( 0 ).setSize( query.getLimit() );
 
+
             for ( Query.SortPredicate sp : query.getSortPredicates() ) {
 
                 final SortOrder order;
@@ -436,34 +449,44 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
                     order = SortOrder.DESC;
                 }
 
-                // we do not know the type of the "order by" property and so we do not know
what
-                // type prefix to use. So, here we add an order by clause for every possible
type
-                // that you can order by: string, number and boolean and we ask ElasticSearch
-                // to ignore any fields that are not present.
-
-                final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
-                final FieldSortBuilder stringSort = SortBuilders.fieldSort( stringFieldName
)
-                        .order( order ).ignoreUnmapped( true );
-                srb.addSort( stringSort );
-
-                logger.debug( "   Sort: {} order by {}", stringFieldName, order.toString()
);
-
-                final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
-                final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName
)
-                        .order( order ).ignoreUnmapped( true );
-                srb.addSort( numberSort );
-                logger.debug( "   Sort: {} order by {}", numberFieldName, order.toString()
);
-
-                final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
-                final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName
)
-                        .order( order ).ignoreUnmapped( true );
-                srb.addSort( booleanSort );
-                logger.debug( "   Sort: {} order by {}", booleanFieldName, order.toString()
);
+
+                //By default we want to order geo location queries to be returned by the
order defined above first.
+                /**
+                 * I'm not sure there is any way to build the sort without traversing the
tree.
+                 */
+                if(fb instanceof GeoDistanceFilterBuilder){
+                    srb.addSort(queryVisitor.getGeoDistanceSortBuilder().order( SortOrder.ASC
).unit( DistanceUnit.KILOMETERS ).geoDistance(
+                        GeoDistance.SLOPPY_ARC ));
+                    logger.info( "  Geo Sort: {} order by {}", sp.getPropertyName(), order.toString()
);
+
+                }
+                else {
+                    // we do not know the type of the "order by" property and so we do not
know what
+                    // type prefix to use. So, here we add an order by clause for every possible
type
+                    // that you can order by: string, number and boolean and we ask ElasticSearch
+                    // to ignore any fields that are not present.
+                    final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
+                    final FieldSortBuilder stringSort = SortBuilders.fieldSort( stringFieldName
).order( order ).ignoreUnmapped( true );
+                    srb.addSort( stringSort );
+
+                    logger.debug( "   Sort: {} order by {}", stringFieldName, order.toString()
);
+
+                    final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
+                    final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName
).order( order ).ignoreUnmapped( true );
+                    srb.addSort( numberSort );
+                    logger.debug( "   Sort: {} order by {}", numberFieldName, order.toString()
);
+
+                    final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
+                    final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName
).order( order )
+                                                                     .ignoreUnmapped( true
);
+                    srb.addSort( booleanSort );
+                    logger.debug( "   Sort: {} order by {}", booleanFieldName, order.toString()
);
+                }
             }
 
 
             if ( logger.isDebugEnabled() ) {
-                logger.debug( "Searching index (read alias): {}\n  scope: {} \n type: {}\n
  query: {} ",
+                logger.info( "Searching index (read alias): {}\n  scope: {} \n type: {}\n
  query: {} ",
                     this.alias.getReadAlias(), context, entityTypes, srb );
             }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/65c5e22d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index f012bab..2e7b174 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -30,6 +30,8 @@ import org.elasticsearch.index.query.FilterBuilder;
 import org.elasticsearch.index.query.FilterBuilders;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -58,7 +60,7 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PR
 
 
 /**
- * Visits tree of  parsed Query operands and populates 
+ * Visits tree of  parsed Query operands and populates
  * ElasticSearch QueryBuilder that represents the query.
  */
 public class EsQueryVistor implements QueryVisitor {
@@ -66,8 +68,9 @@ public class EsQueryVistor implements QueryVisitor {
 
     Stack<QueryBuilder> stack = new Stack<QueryBuilder>();
     List<FilterBuilder> filterBuilders = new ArrayList<FilterBuilder>();
+    GeoDistanceSortBuilder geoSortBuilder = null;
+
 
-    
     @Override
     public void visit( AndOperand op ) throws IndexException {
 
@@ -157,14 +160,14 @@ public class EsQueryVistor implements QueryVisitor {
         Object value = op.getLiteral().getValue();
 
         BoolQueryBuilder qb = QueryBuilders.boolQuery(); // let's do a boolean OR
-        qb.minimumNumberShouldMatch(1); 
+        qb.minimumNumberShouldMatch(1);
 
         // field is an entity/array that needs no name prefix
         qb = qb.should( QueryBuilders.matchQuery( name, value ) );
 
         // OR field is a string and needs the prefix on the name
         qb = qb.should( QueryBuilders.matchQuery( addPrefix( value.toString(), name, true),
value));
-        
+
         stack.push( qb );
     }
 
@@ -186,7 +189,8 @@ public class EsQueryVistor implements QueryVisitor {
         FilterBuilder fb = FilterBuilders.geoDistanceFilter( name )
            .lat( lat ).lon( lon ).distance( distance, DistanceUnit.METERS );
         filterBuilders.add( fb );
-    } 
+        geoSortBuilder = SortBuilders.geoDistanceSort( "location" ).point( lat,lon );
+    }
 
 
     @Override
@@ -222,19 +226,19 @@ public class EsQueryVistor implements QueryVisitor {
             qb.minimumNumberShouldMatch(1);
 
             // field is an entity/array that does not need a prefix on its name
-            // TODO is this right now that we've updated our doc structure?  
+            // TODO is this right now that we've updated our doc structure?
             // Should this be "must" instead of should?
             qb = qb.should( QueryBuilders.wildcardQuery( name, svalue ) );
-           
+
             // or field is just a string that does need a prefix
             if ( svalue.indexOf("*") != -1 ) {
                 qb = qb.should( QueryBuilders.wildcardQuery( addPrefix( value, name ), svalue
) );
             } else {
                 qb = qb.should( QueryBuilders.termQuery(     addPrefix( value, name ), value
));
-            } 
+            }
             stack.push( qb );
             return;
-        } 
+        }
 
         // assume all other types need prefix
         stack.push( QueryBuilders.termQuery( addPrefix( value, name ), value ));
@@ -276,7 +280,7 @@ public class EsQueryVistor implements QueryVisitor {
         if ( parts.length > 1 ) {
             name = parts[ parts.length - 1 ];
         }
-        
+
         if ( value instanceof String && analyzed ) {
             name = addAnalyzedStringPrefix( name );
 
@@ -293,12 +297,12 @@ public class EsQueryVistor implements QueryVisitor {
             name = addStringPrefix( name );
         }
 
-        // re-create nested property name 
+        // re-create nested property name
         if ( parts.length > 1 ) {
             parts[parts.length - 1] = name;
             Joiner joiner = Joiner.on(".").skipNulls();
             return joiner.join(parts);
-        } 
+        }
 
         return name;
     }
@@ -307,34 +311,34 @@ public class EsQueryVistor implements QueryVisitor {
     private String addAnalyzedStringPrefix( String name ) {
         if ( name.startsWith( ANALYZED_STRING_PREFIX ) ) {
             return name;
-        }  
+        }
         return ANALYZED_STRING_PREFIX + name;
-    } 
-   
+    }
+
 
     private String addStringPrefix( String name ) {
         if ( name.startsWith( STRING_PREFIX ) ) {
             return name;
-        } 
+        }
         return STRING_PREFIX + name;
-    } 
-   
+    }
+
 
     private String addNumberPrefix( String name ) {
         if ( name.startsWith( NUMBER_PREFIX ) ) {
             return name;
-        } 
+        }
         return NUMBER_PREFIX + name;
-    } 
-   
+    }
+
 
     private String addBooleanPrefix( String name ) {
         if ( name.startsWith( BOOLEAN_PREFIX ) ) {
             return name;
-        } 
+        }
         return BOOLEAN_PREFIX + name;
-    } 
-   
+    }
+
 
     @Override
     public QueryBuilder getQueryBuilder() {
@@ -354,14 +358,19 @@ public class EsQueryVistor implements QueryVisitor {
 			for ( FilterBuilder fb : filterBuilders ) {
 				if ( andFilter == null ) {
 					andFilter = FilterBuilders.andFilter( fb );
-				} else {	
+				} else {
 					andFilter = FilterBuilders.andFilter( andFilter, fb );
 				}
-			}	
+			}
 
 		} else if ( !filterBuilders.isEmpty() ) {
 			return filterBuilders.get(0);
 		}
 		return null;
 	}
+
+    @Override
+    public GeoDistanceSortBuilder getGeoDistanceSortBuilder(){
+        return geoSortBuilder;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/65c5e22d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index da68772..ed6d19e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -103,6 +103,7 @@ public class Query {
     private List<CounterFilterPredicate> counterFilters;
     private String collection;
     private String ql;
+    private QueryVisitor visitor;
 
     private static ObjectMapper mapper = new ObjectMapper();
 
@@ -146,27 +147,7 @@ public class Query {
     public QueryBuilder createQueryBuilder( final String context ) {
 
 
-        QueryBuilder queryBuilder = null;
-
-
-        //we have a root operand.  Translate our AST into an ES search
-        if ( getRootOperand() != null ) {
-            // In the case of geo only queries, this will return null into the query builder.
-            // Once we start using tiles, we won't need this check any longer, since a geo
query
-            // will return a tile query + post filter
-            QueryVisitor v = new EsQueryVistor();
-
-            try {
-                getRootOperand().visit( v );
-            }
-            catch ( IndexException ex ) {
-                throw new RuntimeException( "Error building ElasticSearch query", ex );
-            }
-
-
-            queryBuilder = v.getQueryBuilder();
-        }
-
+        QueryBuilder queryBuilder = getQueryVisitor().getQueryBuilder();
 
          // Add our filter for context to our query for fast execution.
          // Fast because it utilizes bitsets internally. See this post for more detail.
@@ -188,23 +169,20 @@ public class Query {
     }
 
 
-	public FilterBuilder createFilterBuilder() {
-	    FilterBuilder filterBuilder = null;
+    public QueryVisitor getQueryVisitor() {
+
+        QueryVisitor v = new EsQueryVistor();
 
         if ( getRootOperand() != null ) {
-            QueryVisitor v = new EsQueryVistor();
             try {
                 getRootOperand().visit( v );
 
             } catch ( IndexException ex ) {
                 throw new RuntimeException( "Error building ElasticSearch query", ex );
             }
-            filterBuilder = v.getFilterBuilder();
         }
-
-        return filterBuilder;
-	}
-
+        return v;
+    }
 
     /**
      * Create a query instance from the QL.  If the string is null, return an empty query

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/65c5e22d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
index d295c92..943718c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
@@ -24,6 +24,7 @@ import org.apache.usergrid.persistence.index.exceptions.NoIndexException;
 import org.apache.usergrid.persistence.index.exceptions.IndexException;
 import org.elasticsearch.index.query.FilterBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
 
 
 /**
@@ -93,10 +94,12 @@ public interface QueryVisitor {
      */
     public void visit( GreaterThanEqual op ) throws NoIndexException;
 
-    /** 
+    /**
      * Returns resulting query builder.
      */
     public QueryBuilder getQueryBuilder();
 
 	public FilterBuilder getFilterBuilder();
+
+    public GeoDistanceSortBuilder getGeoDistanceSortBuilder();
 }


Mime
View raw message