usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From toddn...@apache.org
Subject [2/2] incubator-usergrid git commit: Updates constants.
Date Fri, 07 Nov 2014 00:43:39 GMT
Updates constants.

Fixes scope in query

Need to finish version query


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ef77a5a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ef77a5a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ef77a5a4

Branch: refs/heads/es-type-updates
Commit: ef77a5a4eeca44496bf0ced75cf662f49040cba8
Parents: 2db132c
Author: Todd Nine <tnine@apigee.com>
Authored: Thu Nov 6 16:55:13 2014 -0700
Committer: Todd Nine <tnine@apigee.com>
Committed: Thu Nov 6 17:43:18 2014 -0700

----------------------------------------------------------------------
 .../index/impl/EsEntityIndexBatchImpl.java      |  11 +-
 .../index/impl/EsEntityIndexImpl.java           | 126 ++-
 .../persistence/index/impl/EsQueryVistor.java   |   3 +-
 .../persistence/index/impl/IndexingUtils.java   | 116 +--
 .../usergrid/persistence/index/query/Query.java |  28 +-
 .../impl/EntityConnectionIndexImplTest.java     |  45 +-
 .../persistence/index/impl/EntityIndexTest.java | 785 ++++++++++---------
 .../persistence/index/impl/EsRunner.java        |   2 +-
 .../persistence/query/tree/GrammarTreeTest.java |   2 +-
 9 files changed, 590 insertions(+), 528 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 5a19d1e..01f22c0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -61,8 +61,8 @@ import com.google.common.base.Joiner;
 
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT_FIELDNAME;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
@@ -116,7 +116,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         if ( log.isDebugEnabled() ) {
             log.debug( "Indexing entity {}:{} in scope\n   app {}\n   "
-                    + "owner {}\n   name {}\n   type {} \n scope type{}", new Object[] {
+                    + "owner {}\n   name {}\n   type {} \n scope type {}", new Object[] {
                     entity.getId().getType(), entity.getId().getUuid(), applicationScope.getApplication(),
                     indexScope.getOwner(), indexScope.getName(), entityType, context
             } );
@@ -255,10 +255,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
         final Map entityMap = entityToMap( entity );
 
         //add the context for filtering later
-        entityMap.put( ENTITY_CONTEXT, context );
+        entityMap.put( ENTITY_CONTEXT_FIELDNAME, context );
 
         //but the fieldname
-        entityMap.put( ENTITYID_FIELDNAME, entity.getId().getUuid().toString().toLowerCase() );
+        //we have to prefix because we use query equality to seek this later.  TODO see if we can make this more declarative
+        entityMap.put( STRING_PREFIX+ ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase() );
 
         return entityMap;
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 7378ecd..d7955c4 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -33,17 +33,14 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequestBuilder;
 import org.elasticsearch.client.AdminClient;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.TermFilterBuilder;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndexMissingException;
 import org.elasticsearch.search.SearchHit;
@@ -54,8 +51,6 @@ import org.elasticsearch.search.sort.SortOrder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.lucene.queryparser.xml.FilterBuilderFactory;
-
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.util.Health;
 import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -76,9 +71,9 @@ import com.google.inject.Inject;
 import com.google.inject.assistedinject.Assisted;
 
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.DOC_ID_SEPARATOR_SPLITTER;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
 
 
@@ -148,12 +143,12 @@ public class EsEntityIndexImpl implements EntityIndex {
             final int numberOfShards = config.getNumberOfShards();
             final int numberOfReplicas = config.getNumberOfReplicas();
 
-            Settings settings = ImmutableSettings.settingsBuilder()
-            .put("index.number_of_shards", numberOfShards )
-            .put("index.number_of_replicas", numberOfReplicas).build();
+            Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards )
+                                                 .put( "index.number_of_replicas", numberOfReplicas ).build();
 
 
-            final  CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).setSettings( settings ).execute().actionGet();
+            final CreateIndexResponse cir =
+                    admin.indices().prepareCreate( indexName ).setSettings( settings ).execute().actionGet();
 
             logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
 
@@ -175,8 +170,8 @@ public class EsEntityIndexImpl implements EntityIndex {
 
 
     /**
-     * Tests writing a document to a new index to ensure it's working correctly. 
-     * See this post: http://s.apache.org/index-missing-exception
+     * Tests writing a document to a new index to ensure it's working correctly. See this post:
+     * http://s.apache.org/index-missing-exception
      */
     private void testNewIndex() {
 
@@ -188,18 +183,17 @@ public class EsEntityIndexImpl implements EntityIndex {
             public boolean doOp() {
                 final String tempId = UUIDGenerator.newTimeUUID().toString();
 
-                esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId )
-                        .setSource( DEFAULT_PAYLOAD ).get();
+                esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
+                          .get();
 
-                logger.info( "Successfully created new document with docId {} in index {} and type {}", 
-                        tempId, indexName, VERIFY_TYPE );
+                logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId,
+                        indexName, VERIFY_TYPE );
 
                 // delete all types, this way if we miss one it will get cleaned up
                 esProvider.getClient().prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE )
-                        .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
+                          .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
 
-                logger.info( "Successfully deleted all documents in index {} and type {}", 
-                        indexName, VERIFY_TYPE );
+                logger.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE );
 
                 return true;
             }
@@ -218,10 +212,11 @@ public class EsEntityIndexImpl implements EntityIndex {
         XContentBuilder xcb =
                 IndexingUtils.createDoubleStringIndexMapping( XContentFactory.jsonBuilder(), "_default_" );
 
-        PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices().preparePutTemplate( "usergrid_template" )
-                                              .setTemplate( config.getIndexPrefix() + "*" ).addMapping( "_default_",
+        PutIndexTemplateResponse pitr =
+                esProvider.getClient().admin().indices().preparePutTemplate( "usergrid_template" )
+                          .setTemplate( config.getIndexPrefix() + "*" ).addMapping( "_default_",
                         xcb ) // set mapping as the default for all types
-                .execute().actionGet();
+                        .execute().actionGet();
     }
 
 
@@ -237,40 +232,40 @@ public class EsEntityIndexImpl implements EntityIndex {
         final String context = IndexingUtils.createContextName( indexScope );
         final String[] entityTypes = searchTypes.getTypeNames();
 
-        QueryBuilder qb = query.createQueryBuilder();
+        QueryBuilder qb = query.createQueryBuilder(context);
 
-        if ( logger.isDebugEnabled() ) {
-            logger.debug( "Searching index {}\n  scope{} \n type {}\n   query {} limit {}", new Object[] {
-                    this.indexName, context, entityTypes, qb.toString().replace( "\n", " " ), query.getLimit()
-            } );
-        }
 
         SearchResponse searchResponse;
 
         if ( query.getCursor() == null ) {
-            SearchRequestBuilder srb =
-                    esProvider.getClient().prepareSearch( indexName ).setTypes( entityTypes ).setScroll(
-                            cursorTimeout + "m" )
-                          .setQuery( qb );
+            SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName ).setTypes( entityTypes )
+                                                 .setScroll( cursorTimeout + "m" ).setQuery( qb );;
 
-            final TermFilterBuilder contextFilter = FilterBuilders.termFilter( IndexingUtils.ENTITY_CONTEXT, context );
 
-            final FilterBuilder fb = query.createFilterBuilder();
+            //we must add a "must" operation to our entity context.
+
 
+            //set our final query
+
+
+            if ( logger.isDebugEnabled() ) {
+                        logger.debug( "Searching index {}\n  scope{} \n type {}\n   query {} limit {}", new Object[] {
+                                this.indexName, context, entityTypes, qb.toString().replace( "\n", " " ), query.getLimit()
+                        } );
+                    }
+
+
+
+            final FilterBuilder fb = query.createFilterBuilder();
 
 
             //we have post filters, apply them
             if ( fb != null ) {
-//                final FilterBuilder postFilters = FilterBuilders.andFilter(fb, contextFilter  );
-
                 logger.debug( "   Filter: {} ", fb.toString() );
-//                srb = srb.setPostFilter( postFilters );
-            }
-            //no other post filters, just the types
-            else{
-//                srb.setPostFilter( contextFilter );
+                srb = srb.setPostFilter( fb );
             }
 
+
             srb = srb.setFrom( 0 ).setSize( query.getLimit() );
 
             for ( Query.SortPredicate sp : query.getSortPredicates() ) {
@@ -289,20 +284,20 @@ public class EsEntityIndexImpl implements EntityIndex {
                 // to ignore any fields that are not present.
 
                 final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
-                final FieldSortBuilder stringSort = SortBuilders.fieldSort( stringFieldName )
-                        .order( order ).ignoreUnmapped( true );
+                final FieldSortBuilder stringSort =
+                        SortBuilders.fieldSort( stringFieldName ).order( order ).ignoreUnmapped( true );
                 srb.addSort( stringSort );
                 logger.debug( "   Sort: {} order by {}", stringFieldName, order.toString() );
 
                 final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
-                final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName )
-                        .order( order ).ignoreUnmapped( true );
+                final FieldSortBuilder numberSort =
+                        SortBuilders.fieldSort( numberFieldName ).order( order ).ignoreUnmapped( true );
                 srb.addSort( numberSort );
                 logger.debug( "   Sort: {} order by {}", numberFieldName, order.toString() );
 
                 final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
-                final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName )
-                        .order( order ).ignoreUnmapped( true );
+                final FieldSortBuilder booleanSort =
+                        SortBuilders.fieldSort( booleanFieldName ).order( order ).ignoreUnmapped( true );
                 srb.addSort( booleanSort );
                 logger.debug( "   Sort: {} order by {}", booleanFieldName, order.toString() );
             }
@@ -329,7 +324,8 @@ public class EsEntityIndexImpl implements EntityIndex {
             }
             logger.debug( "Executing query with cursor: {} ", scrollId );
 
-            SearchScrollRequestBuilder ssrb = esProvider.getClient().prepareSearchScroll( scrollId ).setScroll( cursorTimeout + "m" );
+            SearchScrollRequestBuilder ssrb =
+                    esProvider.getClient().prepareSearchScroll( scrollId ).setScroll( cursorTimeout + "m" );
 
             try {
                 searchResponse = ssrb.execute().actionGet();
@@ -351,7 +347,7 @@ public class EsEntityIndexImpl implements EntityIndex {
 
         for ( SearchHit hit : hits.getHits() ) {
 
-            String[] idparts = hit.getId().split( DOC_ID_SEPARATOR_SPLITTER );
+            String[] idparts = hit.getId().split( SPLITTER );
             String id = idparts[0];
             String type = idparts[1];
             String version = idparts[2];
@@ -386,7 +382,7 @@ public class EsEntityIndexImpl implements EntityIndex {
                     return true;
                 }
                 catch ( IndexMissingException e ) {
-                    logger.error( "Unable to refresh index after create. Waiting before sleeping.", e);
+                    logger.error( "Unable to refresh index after create. Waiting before sleeping.", e );
                     throw e;
                 }
             }
@@ -401,7 +397,7 @@ public class EsEntityIndexImpl implements EntityIndex {
     @Override
     public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
         Query query = new Query();
-        query.addEqualityFilter( ENTITYID_FIELDNAME, id.getUuid().toString() );
+        query.addEqualityFilter( ENTITYID_ID_FIELDNAME, IndexingUtils.idString( id ).toLowerCase() );
         CandidateResults results = search( scope, SearchTypes.fromTypes( id.getType() ), query );
         return results;
     }
@@ -424,7 +420,6 @@ public class EsEntityIndexImpl implements EntityIndex {
 
     /**
      * Do the retry operation
-     * @param operation
      */
     private void doInRetry( final RetryOperation operation ) {
         for ( int i = 0; i < MAX_WAITS; i++ ) {
@@ -456,13 +451,13 @@ public class EsEntityIndexImpl implements EntityIndex {
     public Health getClusterHealth() {
 
         try {
-            ClusterHealthResponse chr = esProvider.getClient().admin().cluster()
-                .health( new ClusterHealthRequest() ).get();
+            ClusterHealthResponse chr =
+                    esProvider.getClient().admin().cluster().health( new ClusterHealthRequest() ).get();
             return Health.valueOf( chr.getStatus().name() );
-        } 
-        catch (Exception ex) {
-            logger.error("Error connecting to ElasticSearch", ex);
-        } 
+        }
+        catch ( Exception ex ) {
+            logger.error( "Error connecting to ElasticSearch", ex );
+        }
 
         // this is bad, red alert!
         return Health.RED;
@@ -474,15 +469,16 @@ public class EsEntityIndexImpl implements EntityIndex {
      */
     @Override
     public Health getIndexHealth() {
-        
+
         try {
             ClusterHealthResponse chr = esProvider.getClient().admin().cluster()
-                .health( new ClusterHealthRequest( new String[] { indexName } ) ).get();
+                                                  .health( new ClusterHealthRequest( new String[] { indexName } ) )
+                                                  .get();
             return Health.valueOf( chr.getStatus().name() );
-        } 
-        catch (Exception ex) {
-            logger.error("Error connecting to ElasticSearch", ex);
-        } 
+        }
+        catch ( Exception ex ) {
+            logger.error( "Error connecting to ElasticSearch", ex );
+        }
 
         // this is bad, red alert!
         return Health.RED;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index 0afe992..0e4b0ad 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -219,9 +219,10 @@ public class EsQueryVistor implements QueryVisitor {
             String svalue = (String)value;
 
             BoolQueryBuilder qb = QueryBuilders.boolQuery();  // let's do a boolean OR
-            qb.minimumNumberShouldMatch(1); 
+            qb.minimumNumberShouldMatch(1);
 
             // field is an entity/array that does not need a prefix on its name
+            //TODO is this right now that we've updated our doc structure?  Should this be "must" instead of should?
             qb = qb.should( QueryBuilders.wildcardQuery( name, svalue ) );
            
             // or field is just a string that does need a prefix

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index c8e0b86..c7592bc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -37,22 +37,21 @@ public class IndexingUtils {
     public static final String NUMBER_PREFIX = "nu_";
     public static final String BOOLEAN_PREFIX = "bu_";
 
-    public static final String ENTITYID_FIELDNAME = "entityId";
 
-    public static final String DOC_ID_SEPARATOR = "|";
-    public static final String DOC_ID_SEPARATOR_SPLITTER = "\\|";
 
-    // These are not allowed in document type names: _ . , | #
-    public static final String DOC_TYPE_SEPARATOR = "^";
-
-    public static final String INDEX_NAME_SEPARATOR = "^";
+    public static final String SPLITTER = "\\__";
 
-    public static final String ENTITY_CONTEXT = "_context";
+    // These are not allowed in document type names: _ . , | #
+    public static final String SEPARATOR = "__";
 
     /**
-     * To be used when we want to search all types within a scope
+     * Reserved UG fields.
      */
-    public static final String ALL_TYPES = "ALL";
+
+    public static final String ENTITY_CONTEXT_FIELDNAME = "ug_context";
+
+    public static final String ENTITYID_ID_FIELDNAME = "ug_entityId";
+
 
 
     /**
@@ -62,13 +61,35 @@ public class IndexingUtils {
       */
      public static String createContextName( IndexScope scope ) {
          StringBuilder sb = new StringBuilder();
-         sb.append( scope.getOwner().getUuid() ).append(DOC_TYPE_SEPARATOR);
-         sb.append( scope.getOwner().getType() ).append(DOC_TYPE_SEPARATOR);
+         idString(sb, scope.getOwner());
+         sb.append( SEPARATOR );
          sb.append( scope.getName() );
          return sb.toString();
      }
 
 
+    /**
+     * Append the id to the string
+     * @param builder
+     * @param id
+     */
+    public static final void idString(final StringBuilder builder, final Id id){
+        builder.append( id.getUuid() ).append( SEPARATOR )
+                .append(id.getType());
+    }
+
+
+    /**
+     * Turn the id into a string
+     * @param id
+     * @return
+     */
+    public static final String idString(final Id id){
+        final StringBuilder sb = new StringBuilder(  );
+        idString(sb, id);
+        return sb.toString();
+    }
+
 
     /**
      * Create the index name based on our prefix+appUUID+AppType
@@ -79,9 +100,8 @@ public class IndexingUtils {
     public static String createIndexName(
             String prefix, ApplicationScope applicationScope) {
         StringBuilder sb = new StringBuilder();
-        sb.append( prefix ).append(INDEX_NAME_SEPARATOR);
-        sb.append( applicationScope.getApplication().getUuid() ).append(INDEX_NAME_SEPARATOR);
-        sb.append( applicationScope.getApplication().getType() );
+        sb.append( prefix ).append( SEPARATOR );
+        idString( sb, applicationScope.getApplication() );
         return sb.toString();
     }
 
@@ -92,8 +112,8 @@ public class IndexingUtils {
      * @param entity
      * @return
      */
-    public static String createIndexDocId(final Entity entity, final String scopeType) {
-        return createIndexDocId(entity.getId(), entity.getVersion(), scopeType);
+    public static String createIndexDocId(final Entity entity, final String context) {
+        return createIndexDocId(entity.getId(), entity.getVersion(), context);
     }
 
 
@@ -101,14 +121,15 @@ public class IndexingUtils {
      * Create the doc Id. This is the entitie's type + uuid + version
      * @param entityId
      * @param version
+     * @para context The context it's indexed in
      * @return
      */
-    public static String createIndexDocId(final Id entityId, final UUID version, final String scopeType) {
+    public static String createIndexDocId(final Id entityId, final UUID version, final String context) {
         StringBuilder sb = new StringBuilder();
-        sb.append( entityId.getUuid() ).append(DOC_ID_SEPARATOR);
-        sb.append( entityId.getType() ).append(DOC_ID_SEPARATOR);
-        sb.append( version.toString() ).append( DOC_ID_SEPARATOR );
-        sb.append( scopeType);
+        idString(sb, entityId);
+        sb.append( SEPARATOR );
+        sb.append( version.toString() ).append( SEPARATOR );
+        sb.append( context);
         return sb.toString();
     }
 
@@ -135,25 +156,41 @@ public class IndexingUtils {
 
                     .startArray( "dynamic_templates" )
 
-                        // any string with field name that starts with sa_ gets analyzed
+                           //we need most specific mappings first since it's a stop on match algorithm
+
                         .startObject()
-                            .startObject( "template_1" )
-                                .field( "match", ANALYZED_STRING_PREFIX + "*" )
-                                .field( "match_mapping_type", "string" )
+                        .startObject( "context_template" )
+                            .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
+                                 .field( "match_mapping_type", "string" )
+                                        .startObject( "mapping" ).field( "type", "string" )
+                                             .field( "index", "not_analyzed" )
+                                        .endObject()
+                                 .endObject()
+                             .endObject()
+
+
+                        .startObject()
+                        .startObject( "context_template" )
+                            .field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
+                            .field( "match_mapping_type", "string" )
                                 .startObject( "mapping" ).field( "type", "string" )
-                                    .field( "index", "analyzed" )
+                                    .field( "index", "not_analyzed" ).endObject()
                                 .endObject()
-                            .endObject()
+                         .endObject()
+
+                        // any string with field name that starts with sa_ gets analyzed
+                        .startObject()
+                            .startObject( "template_1" ).field( "match", ANALYZED_STRING_PREFIX + "*" )
+                            .field( "match_mapping_type", "string" ).startObject( "mapping" ).field( "type", "string" )
+                            .field( "index", "analyzed" ).endObject().endObject()
                         .endObject()
 
-                            // all other strings are not analyzed
+                                // all other strings are not analyzed
                         .startObject()
                             .startObject( "template_2" )
-                //todo, should be string prefix, remove 2 field mapping
-                                .field( "match", "*" )
-                                .field( "match_mapping_type", "string" )
-                                .startObject( "mapping" )
-                                    .field( "type", "string" )
+                                    //todo, should be string prefix, remove 2 field mapping
+                                .field( "match", "*" ).field( "match_mapping_type", "string" ).startObject( "mapping" )
+                                .field( "type", "string" )
                                     .field( "index", "not_analyzed" )
                                 .endObject()
                             .endObject()
@@ -169,16 +206,7 @@ public class IndexingUtils {
                             .endObject()
                         .endObject()
 
-                //types for context direct string matching
-                .startObject( "context_template" )
-                        .field( "match", IndexingUtils.ENTITY_CONTEXT )
-                        .field( "match_mapping_type", "string" )
-                        .startObject( "mapping" )
-                            .field( "type", "string" )
-                            .field( "index", "not_analyzed" )
-                        .endObject()
-                    .endObject()
-                .endObject()
+
 
                     .endArray()
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index 778134e..07bee44 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -43,6 +43,7 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.usergrid.persistence.index.exceptions.IndexException;
 import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
 import org.apache.usergrid.persistence.index.impl.EsQueryVistor;
+import org.apache.usergrid.persistence.index.impl.IndexingUtils;
 import org.apache.usergrid.persistence.index.query.tree.AndOperand;
 import org.apache.usergrid.persistence.index.query.tree.ContainsOperand;
 import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer;
@@ -142,10 +143,23 @@ public class Query {
     }
 
 
-    public QueryBuilder createQueryBuilder() {
+    public QueryBuilder createQueryBuilder(final String context) {
 
-        QueryBuilder queryBuilder = null;
 
+        QueryBuilder queryBuilder;
+
+
+        /**
+         * Add our filter for context to our query for fast execution.  Fast because it utilizes bitsets
+         * internally. See this post for more detail.
+         * http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
+         */
+
+
+
+
+
+        //we have a root operand.  Translate our AST into an ES search
         if ( getRootOperand() != null ) {
             QueryVisitor v = new EsQueryVistor();
             try {
@@ -154,11 +168,15 @@ public class Query {
             } catch ( IndexException ex ) {
                 throw new RuntimeException( "Error building ElasticSearch query", ex );
             }
-            queryBuilder = v.getQueryBuilder();
+
+            // TODO evaluate performance when it's an all query.  Do we need to put the context term first for performance?
+            queryBuilder = QueryBuilders.boolQuery().must(  v.getQueryBuilder() )
+                                        .must( QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) );
         } 
 
-		if ( queryBuilder == null ) {
-            queryBuilder = QueryBuilders.matchAllQuery();
+        //nothing was specified ensure we specify the context in the search
+		else {
+            queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
 		}
 
         return queryBuilder;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index c1c075e..d2423bb 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -40,6 +40,7 @@ import org.apache.usergrid.persistence.index.EntityIndexFactory;
 import org.apache.usergrid.persistence.index.IndexScope;
 import org.apache.usergrid.persistence.index.SearchTypes;
 import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
 import org.apache.usergrid.persistence.model.entity.Entity;
@@ -51,6 +52,7 @@ import com.google.inject.Inject;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
 
 
 @RunWith( EsRunner.class )
@@ -59,15 +61,9 @@ public class EntityConnectionIndexImplTest extends BaseIT {
 
     private static final Logger log = LoggerFactory.getLogger( EntityConnectionIndexImplTest.class );
 
-    @ClassRule
-    public static ElasticSearchResource es = new ElasticSearchResource();
+//    @ClassRule
+//    public static ElasticSearchResource es = new ElasticSearchResource();
 
-    @ClassRule
-    public static CassandraRule cass = new CassandraRule();
-
-    @Inject
-    @Rule
-    public MigrationManagerRule migrationManagerRule;
 
     @Inject
     public EntityIndexFactory ecif;
@@ -121,7 +117,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
         IndexScope searchScope = new IndexScopeImpl( personId, "likes" );
 
         //create another scope we index in, want to be sure these scopes are filtered
-        IndexScope otherIndexScope = new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(), "person" ), "likes" );
+        IndexScope otherIndexScope = new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(), "animal" ), "likes" );
 
         EntityIndex personLikesIndex = ecif.createEntityIndex( applicationScope );
 
@@ -142,6 +138,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
         batch.index( otherIndexScope, oj );
 
         batch.executeAndRefresh();
+        personLikesIndex.refresh();
 
         // now, let's search for muffins
         CandidateResults likes = personLikesIndex
@@ -167,18 +164,18 @@ public class EntityConnectionIndexImplTest extends BaseIT {
                 SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType() ),
                 Query.fromQL( "select *" ) );
         assertEquals( 3, likes.size() );
-        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
-        assertEquals( egg.getId(), likes.get( 1 ).getId() );
-        assertEquals( oj.getId(), likes.get( 2 ).getId() );
+        assertContains( egg.getId(), likes );
+        assertContains( muffin.getId(), likes );
+        assertContains( oj.getId(), likes );
 
           //now lets search for all explicitly
         likes = personLikesIndex.search( searchScope,
                 SearchTypes.allTypes(),
                 Query.fromQL( "select *" ) );
         assertEquals( 3, likes.size() );
-        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
-        assertEquals( egg.getId(), likes.get( 1 ).getId() );
-        assertEquals( oj.getId(), likes.get( 2 ).getId() );
+        assertContains( egg.getId(), likes );
+        assertContains( muffin.getId(), likes);
+        assertContains( oj.getId(), likes );
 
 
 
@@ -187,16 +184,26 @@ public class EntityConnectionIndexImplTest extends BaseIT {
                 SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType() ),
                 Query.fromQL( "select * where stars = 5" ) );
         assertEquals( 2, likes.size() );
-        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
-        assertEquals( egg.getId(), likes.get( 1 ).getId() );
+        assertContains( egg.getId(), likes);
+        assertContains( muffin.getId(), likes );
 
 
 
         //now search with no types, we should get only the results that match
         likes = personLikesIndex.search( searchScope, SearchTypes.allTypes(), Query.fromQL( "select * where stars = 5" ) );
         assertEquals( 2, likes.size() );
-        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
-        assertEquals( egg.getId(), likes.get( 1 ).getId() );
+        assertContains( egg.getId(), likes);
+        assertContains( muffin.getId(), likes );
+    }
+
+
+    private void assertContains(final Id id, final CandidateResults results ){
+        for( CandidateResult result: results){
+            if(result.getId().equals(id)){
+                return;
+            }
+        }
 
+        fail(String.format("Could not find id %s in candidate results", id));
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 1bbb301..a1240f1 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -45,6 +45,7 @@ import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.EntityIndexBatch;
 import org.apache.usergrid.persistence.index.EntityIndexFactory;
 import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
 import org.apache.usergrid.persistence.index.guice.TestIndexModule;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
@@ -69,393 +70,403 @@ import static org.junit.Assert.assertNotEquals;
 @UseModules({ TestIndexModule.class })
 public class EntityIndexTest extends BaseIT {
 
-//    private static final Logger log = LoggerFactory.getLogger( EntityIndexTest.class );
-//
-//    @ClassRule
-//    public static CassandraRule cass = new CassandraRule();
-//
-//    @Rule
-//    public ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
-//
-//    @Inject
-//    @Rule
-//    public MigrationManagerRule migrationManagerRule;
-//
-//    @Inject
-//    public EntityIndexFactory eif;
-//
-//
-//
-//    @Test
-//    public void testIndex() throws IOException {
-//
-//        final int MAX_ENTITIES = 100;
-//
-//        Id appId = new SimpleId( "application" );
-//
-//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-//
-//        IndexScope indexScope = new IndexScopeImpl( appId, "things" );
-//
-//
-//        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
-//        entityIndex.initializeIndex();
-//
-//        InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
-//        ObjectMapper mapper = new ObjectMapper();
-//        List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} );
-//
-//        int count = 0;
-//        StopWatch timer = new StopWatch();
-//        timer.start();
-//
-//        final EntityIndexBatch batch = entityIndex.createBatch();
-//
-//        for ( Object o : sampleJson ) {
-//
-//            Map<String, Object> item = ( Map<String, Object> ) o;
-//
-//            Entity entity = new Entity( indexScope.getName() );
-//            entity = EntityIndexMapUtils.fromMap( entity, item );
-//            EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
-//
-//            batch.index( indexScope, entity );
-//
-//            if(count %1000 == 0){
-//                batch.execute();
-//            }
-//
-//
-//
-//            if ( count++ > MAX_ENTITIES ) {
-//                break;
-//            }
-//        }
-//
-//        batch.execute();
-//        timer.stop();
-//        log.info( "Total time to index {} entries {}ms, average {}ms/entry",
-//                new Object[] { count, timer.getTime(), timer.getTime() / count } );
-//
-//        entityIndex.refresh();
-//
-//
-//        testQueries( indexScope, entityIndex );
-//    }
-//
-//
-//    @Test
-//    public void testDeindex() {
-//
-//        Id appId = new SimpleId( "application" );
-//
-//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-//
-//        IndexScope indexScope = new IndexScopeImpl( appId, "fastcars", entityType );
-//
-//        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
-//        entityIndex.initializeIndex();
-//
-//        Map entityMap = new HashMap() {{
-//            put( "name", "Ferrari 212 Inter" );
-//            put( "introduced", 1952 );
-//            put( "topspeed", 215 );
-//        }};
-//
-//
-//        Entity entity = EntityIndexMapUtils.fromMap( entityMap );
-//        EntityUtils.setId( entity, new SimpleId( "fastcar" ) );
-//        EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
-//        entityIndex.createBatch().index(indexScope , entity ).executeAndRefresh();
-//
-//        CandidateResults candidateResults = entityIndex.search(indexScope,  Query.fromQL( "name contains 'Ferrari*'" ) );
-//        assertEquals( 1, candidateResults.size() );
-//
-//        entityIndex.createBatch().deindex( indexScope, entity ).execute();
-//
-//        entityIndex.refresh();
-//
-//        candidateResults = entityIndex.search( indexScope, Query.fromQL( "name contains 'Ferrari*'" ) );
-//        assertEquals( 0, candidateResults.size() );
-//    }
-//
-//
-//    private void testQuery(final IndexScope scope, final EntityIndex entityIndex, final String queryString, final int num ) {
-//
-//        StopWatch timer = new StopWatch();
-//        timer.start();
-//        Query query = Query.fromQL( queryString );
-//        query.setLimit( 1000 );
-//        CandidateResults candidateResults = entityIndex.search( scope, query );
-//        timer.stop();
-//
-//        assertEquals( num, candidateResults.size() );
-//        log.debug( "Query time {}ms", timer.getTime() );
-//    }
-//
-//
-//    private void testQueries(final IndexScope scope, final EntityIndex entityIndex ) {
-//
-//
-//        testQuery(scope,  entityIndex, "name = 'Morgan Pierce'", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'morgan pierce'", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Morgan'", 0 );
-//
-//        testQuery(scope,  entityIndex, "name contains 'Morgan'", 1 );
-//
-//        testQuery(scope,  entityIndex, "company > 'GeoLogix'", 64 );
-//
-//        testQuery(scope,  entityIndex, "gender = 'female'", 45 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age > 39", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age > 39 and age < 41", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age > 40", 0 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age >= 40", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age <= 40", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Morgan* '", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Morgan*'", 1 );
-//
-//
-//        // test a couple of array sub-property queries
-//
-//        int totalUsers = 102;
-//
-//        // nobody has a friend named Jack the Ripper
-//        testQuery(scope,  entityIndex, "friends.name = 'Jack the Ripper'", 0 );
-//
-//        // everybody doesn't have a friend named Jack the Ripper
-//        testQuery(scope,  entityIndex, "not (friends.name = 'Jack the Ripper')", totalUsers );
-//
-//        // one person has a friend named Shari Hahn
-//        testQuery(scope,  entityIndex, "friends.name = 'Wendy Moody'", 1 );
-//
-//        // everybody but 1 doesn't have a friend named Shari Hahh
-//        testQuery(scope,  entityIndex, "not (friends.name = 'Shari Hahn')", totalUsers - 1);
-//
-//    }
-//
-//
-//    /**
-//     * Tests that Entity-to-map and Map-to-entity round trip works.
-//     */
-//    @Test
-//    public void testEntityIndexMapUtils() throws IOException {
-//
-//        InputStream is = this.getClass().getResourceAsStream( "/sample-small.json" );
-//        ObjectMapper mapper = new ObjectMapper();
-//        List<Object> contacts = mapper.readValue( is, new TypeReference<List<Object>>() {} );
-//
-//        for ( Object o : contacts ) {
-//
-//            Map<String, Object> map1 = ( Map<String, Object> ) o;
-//
-//            // convert map to entity
-//            Entity entity1 = EntityIndexMapUtils.fromMap( map1 );
-//
-//            // convert entity back to map
-//            Map map2 = EntityIndexMapUtils.toMap( entity1 );
-//
-//            // the two maps should be the same
-//            Map diff = Maps.difference( map1, map2 ).entriesDiffering();
-//            assertEquals( 0, diff.size() );
-//        }
-//    }
-//
-//
-//    @Test
-//    public void getEntityVersions() throws Exception {
-//
-//        Id appId = new SimpleId( "application" );
-//        Id ownerId = new SimpleId( "owner" );
-//
-//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-//
-//        IndexScope indexScope = new IndexScopeImpl( ownerId, "user", entityType );
-//
-//
-//
-//        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
-//        entityIndex.initializeIndex();
-//
-//        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
-//        Map<String, Object> properties = new LinkedHashMap<String, Object>();
-//        properties.put( "username", "edanuff" );
-//        properties.put( "email", "ed@anuff.com" );
-//        properties.put( "middlename", middleName );
-//
-//        Map entityMap = new HashMap() {{
-//            put( "username", "edanuff" );
-//            put( "email", "ed@anuff.com" );
-//            put( "middlename", middleName );
-//        }};
-//
-//        Entity user = EntityIndexMapUtils.fromMap( entityMap );
-//        EntityUtils.setId( user, new SimpleId( "edanuff" ) );
-//        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
-//
-//
-//        final EntityIndexBatch batch = entityIndex.createBatch();
-//
-//        batch.index( indexScope, user );
-//
-//        user.setField( new StringField( "address1", "1782 address st" ) );
-//        batch.index( indexScope, user );
-//        user.setField( new StringField( "address2", "apt 508" ) );
-//        batch.index( indexScope,  user );
-//        user.setField( new StringField( "address3", "apt 508" ) );
-//        batch.index( indexScope,  user );
-//        batch.executeAndRefresh();
-//
-//        CandidateResults results = entityIndex.getEntityVersions(indexScope,  user.getId() );
-//
-//        assertEquals(1,  results.size());
-//        assertEquals( results.get( 0 ).getId(), user.getId() );
-//        assertEquals( results.get(0).getVersion(), user.getVersion());
-//    }
-//
-//
-//    @Test
-//    public void deleteVerification() throws Throwable {
-//
-//        Id appId = new SimpleId( "application" );
-//        Id ownerId = new SimpleId( "owner" );
-//
-//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-//
-//        IndexScope appScope = new IndexScopeImpl( ownerId, "user", entityType );
-//
-//        EntityIndex ei = eif.createEntityIndex( applicationScope );
-//        ei.initializeIndex();
-//
-//        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
-//
-//        Map entityMap = new HashMap() {{
-//            put( "username", "edanuff" );
-//            put( "email", "ed@anuff.com" );
-//            put( "middlename", middleName );
-//        }};
-//
-//        Entity user = EntityIndexMapUtils.fromMap( entityMap );
-//        EntityUtils.setId( user, new SimpleId( "edanuff" ) );
-//        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
-//
-//
-//        EntityIndexBatch batch = ei.createBatch();
-//
-//        batch.index( appScope, user ).executeAndRefresh();
-//        Query query = new Query();
-//        query.addEqualityFilter( "username", "edanuff" );
-//        CandidateResults r = ei.search( appScope, query );
-//        assertEquals( user.getId(), r.get( 0 ).getId() );
-//
-//        batch.deindex(appScope, user.getId(), user.getVersion() ).executeAndRefresh();
-//
-//
-//        // EntityRef
-//        query = new Query();
-//        query.addEqualityFilter( "username", "edanuff" );
-//        r = ei.search(appScope, query );
-//
-//        assertFalse( r.iterator().hasNext() );
-//    }
-//
-//    @Test
-//    public void multiValuedTypes() {
-//
-//        Id appId = new SimpleId( "entityindextest" );
-//        Id ownerId = new SimpleId( "multivaluedtype" );
-//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-//
-//        IndexScope appScope = new IndexScopeImpl( ownerId, "user", entityType );
-//
-//        EntityIndex ei = eif.createEntityIndex( applicationScope );
-//        ei.createBatch();
-//
-//        // Bill has favorites as string, age as string and retirement goal as number
-//        Map billMap = new HashMap() {{
-//            put( "username", "bill" );
-//            put( "email", "bill@example.com" );
-//            put( "age", "thirtysomething");
-//            put( "favorites", "scallops, croquet, wine");
-//            put( "retirementGoal", 100000);
-//        }};
-//        Entity bill = EntityIndexMapUtils.fromMap( billMap );
-//        EntityUtils.setId( bill, new SimpleId( UUIDGenerator.newTimeUUID(), "user"  ) );
-//        EntityUtils.setVersion( bill, UUIDGenerator.newTimeUUID() );
-//
-//        EntityIndexBatch batch = ei.createBatch();
-//
-//        batch.index( appScope,  bill );
-//
-//        // Fred has age as int, favorites as object and retirement goal as object
-//        Map fredMap = new HashMap() {{
-//            put( "username", "fred" );
-//            put( "email", "fred@example.com" );
-//            put( "age", 41 );
-//            put( "favorites", new HashMap<String, Object>() {{
-//                put("food", "cheezewiz");
-//                put("sport", "nascar");
-//                put("beer", "budwizer");
-//            }});
-//            put( "retirementGoal", new HashMap<String, Object>() {{
-//                put("car", "Firebird");
-//                put("home", "Mobile");
-//            }});
-//        }};
-//        Entity fred = EntityIndexMapUtils.fromMap( fredMap );
-//        EntityUtils.setId( fred, new SimpleId( UUIDGenerator.newTimeUUID(), "user"  ) );
-//        EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() );
-//        batch.index( appScope, fred );
-//
-//        batch.executeAndRefresh();
-//
-//        Query query = new Query();
-//        query.addEqualityFilter( "username", "bill" );
-//        CandidateResults r = ei.search( appScope, query );
-//        assertEquals( bill.getId(), r.get( 0 ).getId() );
-//
-//        query = new Query();
-//        query.addEqualityFilter( "username", "fred" );
-//        r = ei.search( appScope,  query );
-//        assertEquals( fred.getId(), r.get( 0 ).getId() );
-//
-//        query = new Query();
-//        query.addEqualityFilter( "age", 41 );
-//        r = ei.search( appScope,  query );
-//        assertEquals( fred.getId(), r.get( 0 ).getId() );
-//
-//        query = new Query();
-//        query.addEqualityFilter( "age", "thirtysomething" );
-//        r = ei.search(  appScope, query );
-//        assertEquals( bill.getId(), r.get( 0 ).getId() );
-//    }
-//
-//
-//    @Test
-//    public void healthTest() {
-//
-//        Id appId = new SimpleId( "application" );
-//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-//
-//        EntityIndex ei = eif.createEntityIndex( applicationScope );
-//
-//        assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() );
-//        assertEquals( "index not be ready yet", Health.RED, ei.getIndexHealth() );
-//
-//        ei.initializeIndex();
-//        ei.refresh();
-//
-//        assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() );
-//        assertNotEquals( "cluster should be ready now", Health.RED, ei.getClusterHealth() );
-//    }
+    private static final Logger log = LoggerFactory.getLogger( EntityIndexTest.class );
+
+    @ClassRule
+    public static CassandraRule cass = new CassandraRule();
+
+    @Rule
+    public ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
+
+    @Inject
+    @Rule
+    public MigrationManagerRule migrationManagerRule;
+
+    @Inject
+    public EntityIndexFactory eif;
+
+
+
+    @Test
+    public void testIndex() throws IOException {
+
+        final int MAX_ENTITIES = 100;
+
+        Id appId = new SimpleId( "application" );
+
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        IndexScope indexScope = new IndexScopeImpl( appId, "things" );
+
+        final String entityType = "thing";
+
+
+        final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+
+
+        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+        entityIndex.initializeIndex();
+
+        InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
+        ObjectMapper mapper = new ObjectMapper();
+        List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} );
+
+        int count = 0;
+        StopWatch timer = new StopWatch();
+        timer.start();
+
+        final EntityIndexBatch batch = entityIndex.createBatch();
+
+        for ( Object o : sampleJson ) {
+
+            Map<String, Object> item = ( Map<String, Object> ) o;
+
+            Entity entity = new Entity( entityType );
+            entity = EntityIndexMapUtils.fromMap( entity, item );
+            EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
+
+            batch.index( indexScope, entity );
+
+            if(count %1000 == 0){
+                batch.execute();
+            }
+
+
+
+            if ( count++ > MAX_ENTITIES ) {
+                break;
+            }
+        }
+
+        batch.execute();
+        timer.stop();
+        log.info( "Total time to index {} entries {}ms, average {}ms/entry",
+                new Object[] { count, timer.getTime(), timer.getTime() / count } );
+
+        entityIndex.refresh();
+
+
+        testQueries( indexScope, searchTypes,  entityIndex );
+    }
+
+
+    @Test
+    public void testDeindex() {
+
+        Id appId = new SimpleId( "application" );
+
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" );
+
+        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+        entityIndex.initializeIndex();
+
+        Map entityMap = new HashMap() {{
+            put( "name", "Ferrari 212 Inter" );
+            put( "introduced", 1952 );
+            put( "topspeed", 215 );
+        }};
+
+
+        Entity entity = EntityIndexMapUtils.fromMap( entityMap );
+        EntityUtils.setId( entity, new SimpleId( "fastcar" ) );
+        EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
+        entityIndex.createBatch().index(indexScope , entity ).executeAndRefresh();
+
+        CandidateResults candidateResults = entityIndex.search( indexScope, SearchTypes.fromTypes(entity.getId().getType()),
+                Query.fromQL( "name contains 'Ferrari*'" ) );
+        assertEquals( 1, candidateResults.size() );
+
+        entityIndex.createBatch().deindex( indexScope, entity ).execute();
+
+        entityIndex.refresh();
+
+        candidateResults = entityIndex.search( indexScope, SearchTypes.fromTypes(entity.getId().getType()), Query.fromQL( "name contains 'Ferrari*'" ) );
+        assertEquals( 0, candidateResults.size() );
+    }
+
+
+    private void testQuery(final IndexScope scope, final SearchTypes searchTypes, final EntityIndex entityIndex, final String queryString, final int num ) {
+
+        StopWatch timer = new StopWatch();
+        timer.start();
+        Query query = Query.fromQL( queryString );
+        query.setLimit( 1000 );
+        CandidateResults candidateResults = entityIndex.search( scope, searchTypes, query );
+        timer.stop();
+
+        assertEquals( num, candidateResults.size() );
+        log.debug( "Query time {}ms", timer.getTime() );
+    }
+
+
+    private void testQueries(final IndexScope scope, SearchTypes searchTypes, final EntityIndex entityIndex ) {
+
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Morgan Pierce'", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'morgan pierce'", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Morgan'", 0 );
+
+        testQuery(scope, searchTypes, entityIndex, "name contains 'Morgan'", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "company > 'GeoLogix'", 64 );
+
+        testQuery(scope, searchTypes, entityIndex, "gender = 'female'", 45 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age > 39", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age > 39 and age < 41", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age > 40", 0 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age >= 40", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age <= 40", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Morgan* '", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Morgan*'", 1 );
+
+
+        // test a couple of array sub-property queries
+
+        int totalUsers = 102;
+
+        // nobody has a friend named Jack the Ripper
+        testQuery(scope, searchTypes, entityIndex, "friends.name = 'Jack the Ripper'", 0 );
+
+        // everybody doesn't have a friend named Jack the Ripper
+        testQuery(scope,  searchTypes,entityIndex, "not (friends.name = 'Jack the Ripper')", totalUsers );
+
+        // one person has a friend named Shari Hahn
+        testQuery(scope, searchTypes, entityIndex, "friends.name = 'Wendy Moody'", 1 );
+
+        // everybody but 1 doesn't have a friend named Shari Hahh
+        testQuery(scope, searchTypes, entityIndex, "not (friends.name = 'Shari Hahn')", totalUsers - 1);
+
+    }
+
+
+    /**
+     * Tests that Entity-to-map and Map-to-entity round trip works.
+     */
+    @Test
+    public void testEntityIndexMapUtils() throws IOException {
+
+        InputStream is = this.getClass().getResourceAsStream( "/sample-small.json" );
+        ObjectMapper mapper = new ObjectMapper();
+        List<Object> contacts = mapper.readValue( is, new TypeReference<List<Object>>() {} );
+
+        for ( Object o : contacts ) {
+
+            Map<String, Object> map1 = ( Map<String, Object> ) o;
+
+            // convert map to entity
+            Entity entity1 = EntityIndexMapUtils.fromMap( map1 );
+
+            // convert entity back to map
+            Map map2 = EntityIndexMapUtils.toMap( entity1 );
+
+            // the two maps should be the same
+            Map diff = Maps.difference( map1, map2 ).entriesDiffering();
+            assertEquals( 0, diff.size() );
+        }
+    }
+
+
+    @Test
+    public void getEntityVersions() throws Exception {
+
+        Id appId = new SimpleId( "application" );
+        Id ownerId = new SimpleId( "owner" );
+
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        IndexScope indexScope = new IndexScopeImpl( ownerId, "users" );
+
+
+
+        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+        entityIndex.initializeIndex();
+
+        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
+        Map<String, Object> properties = new LinkedHashMap<String, Object>();
+        properties.put( "username", "edanuff" );
+        properties.put( "email", "ed@anuff.com" );
+        properties.put( "middlename", middleName );
+
+        Map entityMap = new HashMap() {{
+            put( "username", "edanuff" );
+            put( "email", "ed@anuff.com" );
+            put( "middlename", middleName );
+        }};
+
+        final Id userId = new SimpleId("user");
+
+        Entity user = EntityIndexMapUtils.fromMap( entityMap );
+        EntityUtils.setId( user, userId);
+        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
+
+
+        final EntityIndexBatch batch = entityIndex.createBatch();
+
+        batch.index( indexScope, user );
+
+        user.setField( new StringField( "address1", "1782 address st" ) );
+        batch.index( indexScope, user );
+        user.setField( new StringField( "address2", "apt 508" ) );
+        batch.index( indexScope,  user );
+        user.setField( new StringField( "address3", "apt 508" ) );
+        batch.index( indexScope,  user );
+        batch.executeAndRefresh();
+
+        CandidateResults results = entityIndex.getEntityVersions(indexScope,  user.getId() );
+
+        assertEquals(1,  results.size());
+        assertEquals( results.get( 0 ).getId(), user.getId() );
+        assertEquals( results.get(0).getVersion(), user.getVersion());
+    }
+
+
+    @Test
+    public void deleteVerification() throws Throwable {
+
+        Id appId = new SimpleId( "application" );
+        Id ownerId = new SimpleId( "owner" );
+
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
+
+        EntityIndex ei = eif.createEntityIndex( applicationScope );
+        ei.initializeIndex();
+
+        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
+
+        Map entityMap = new HashMap() {{
+            put( "username", "edanuff" );
+            put( "email", "ed@anuff.com" );
+            put( "middlename", middleName );
+        }};
+
+        Entity user = EntityIndexMapUtils.fromMap( entityMap );
+        EntityUtils.setId( user, new SimpleId( "edanuff" ) );
+        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
+
+
+        EntityIndexBatch batch = ei.createBatch();
+
+        batch.index( appScope, user ).executeAndRefresh();
+        Query query = new Query();
+        query.addEqualityFilter( "username", "edanuff" );
+        CandidateResults r = ei.search( appScope, SearchTypes.fromTypes( "edanuff" ), query );
+        assertEquals( user.getId(), r.get( 0 ).getId() );
+
+        batch.deindex(appScope, user.getId(), user.getVersion() ).executeAndRefresh();
+
+
+        // EntityRef
+        query = new Query();
+        query.addEqualityFilter( "username", "edanuff" );
+        r = ei.search(appScope,SearchTypes.fromTypes( "edanuff" ),  query );
+
+        assertFalse( r.iterator().hasNext() );
+    }
+
+    @Test
+    public void multiValuedTypes() {
+
+        Id appId = new SimpleId( "entityindextest" );
+        Id ownerId = new SimpleId( "multivaluedtype" );
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
+
+        EntityIndex ei = eif.createEntityIndex( applicationScope );
+        ei.createBatch();
+
+        // Bill has favorites as string, age as string and retirement goal as number
+        Map billMap = new HashMap() {{
+            put( "username", "bill" );
+            put( "email", "bill@example.com" );
+            put( "age", "thirtysomething");
+            put( "favorites", "scallops, croquet, wine");
+            put( "retirementGoal", 100000);
+        }};
+        Entity bill = EntityIndexMapUtils.fromMap( billMap );
+        EntityUtils.setId( bill, new SimpleId( UUIDGenerator.newTimeUUID(), "user"  ) );
+        EntityUtils.setVersion( bill, UUIDGenerator.newTimeUUID() );
+
+        EntityIndexBatch batch = ei.createBatch();
+
+        batch.index( appScope,  bill );
+
+        // Fred has age as int, favorites as object and retirement goal as object
+        Map fredMap = new HashMap() {{
+            put( "username", "fred" );
+            put( "email", "fred@example.com" );
+            put( "age", 41 );
+            put( "favorites", new HashMap<String, Object>() {{
+                put("food", "cheezewiz");
+                put("sport", "nascar");
+                put("beer", "budwizer");
+            }});
+            put( "retirementGoal", new HashMap<String, Object>() {{
+                put("car", "Firebird");
+                put("home", "Mobile");
+            }});
+        }};
+        Entity fred = EntityIndexMapUtils.fromMap( fredMap );
+        EntityUtils.setId( fred, new SimpleId( UUIDGenerator.newTimeUUID(), "user"  ) );
+        EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() );
+        batch.index( appScope, fred );
+
+        batch.executeAndRefresh();
+
+        final SearchTypes searchTypes = SearchTypes.fromTypes( "user" );
+
+        Query query = new Query();
+        query.addEqualityFilter( "username", "bill" );
+        CandidateResults r = ei.search( appScope, searchTypes,  query );
+        assertEquals( bill.getId(), r.get( 0 ).getId() );
+
+        query = new Query();
+        query.addEqualityFilter( "username", "fred" );
+        r = ei.search( appScope, searchTypes,  query );
+        assertEquals( fred.getId(), r.get( 0 ).getId() );
+
+        query = new Query();
+        query.addEqualityFilter( "age", 41 );
+        r = ei.search( appScope, searchTypes,  query );
+        assertEquals( fred.getId(), r.get( 0 ).getId() );
+
+        query = new Query();
+        query.addEqualityFilter( "age", "thirtysomething" );
+        r = ei.search(  appScope, searchTypes, query );
+        assertEquals( bill.getId(), r.get( 0 ).getId() );
+    }
+
+
+    @Test
+    public void healthTest() {
+
+        Id appId = new SimpleId( "application" );
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        EntityIndex ei = eif.createEntityIndex( applicationScope );
+
+        assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() );
+        assertEquals( "index not be ready yet", Health.RED, ei.getIndexHealth() );
+
+        ei.initializeIndex();
+        ei.refresh();
+
+        assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() );
+        assertNotEquals( "cluster should be ready now", Health.RED, ei.getClusterHealth() );
+    }
 }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java
index c972851..2521a38 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java
@@ -36,7 +36,7 @@ public class EsRunner extends ITRunner {
         ElasticSearchResource rule = new ElasticSearchResource();
 
         try {
-            rule.before();
+//            rule.before();
         }
         catch ( Throwable throwable ) {
             throw new RuntimeException( "Unable to start ES", throwable );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
index d0e2d11..34185b3 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
@@ -429,7 +429,7 @@ public class GrammarTreeTest {
         assertEquals( 37f, withinOperand.getLatitude().getFloatValue(), 0 );
         assertEquals( -75f, withinOperand.getLongitude().getFloatValue(), 0 );
 
-        QueryBuilder qb = query.createQueryBuilder();
+        QueryBuilder qb = query.createQueryBuilder("testcontext");
     }
 
 


Mime
View raw message