usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sfeld...@apache.org
Subject incubator-usergrid git commit: scroll changes
Date Mon, 20 Apr 2015 23:27:51 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/USERGRID-578 [created] bbbe96316


scroll changes


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bbbe9631
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bbbe9631
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bbbe9631

Branch: refs/heads/USERGRID-578
Commit: bbbe96316c8d5b9c60bd85c8ebef23dcac5412f7
Parents: 8d965b3
Author: Shawn Feldman <sfeldman@apache.org>
Authored: Mon Apr 20 17:27:37 2015 -0600
Committer: Shawn Feldman <sfeldman@apache.org>
Committed: Mon Apr 20 17:27:37 2015 -0600

----------------------------------------------------------------------
 .../persistence/model/entity/SimpleId.java      |  15 ++-
 .../index/ApplicationEntityIndex.java           |   3 +-
 .../persistence/index/CandidateResults.java     |   8 +-
 .../usergrid/persistence/index/SearchTypes.java |   4 +
 .../impl/EsApplicationEntityIndexImpl.java      | 120 ++++++++++---------
 .../impl/SearchRequestBuilderStrategy.java      |   9 +-
 .../persistence/index/impl/EntityIndexTest.java |  25 ++--
 7 files changed, 99 insertions(+), 85 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
index 7ad4ab4..da0380b 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
@@ -33,8 +33,8 @@ import com.google.common.base.Preconditions;
 public class SimpleId implements Id, Serializable {
 
 
-    private final UUID uuid;
-    private final String type;
+    private UUID uuid;
+    private String type;
 
     public SimpleId( final UUID uuid, final String type ) {
         Preconditions.checkNotNull( uuid, "uuid is required" );
@@ -53,18 +53,29 @@ public class SimpleId implements Id, Serializable {
        this(UUIDGenerator.newTimeUUID(), type);
     }
 
+    public SimpleId(  ){
+
+    }
+
+
 
 
     @Override
     public UUID getUuid() {
         return uuid;
     }
+    public void setUuid(UUID val) {
+         uuid = val;
+    }
 
 
     @Override
     public String getType() {
         return type;
     }
+    public String setType(String val) {
+        return type = val;
+    }
 
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index 81b3028..b06cefe 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -44,7 +44,8 @@ public interface ApplicationEntityIndex {
     CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes,
final String query,
                              final int limit );
 
-
+    CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes,
final String query,
+                             final int limit, final int from );
 
     /**
      * Get next page of results from a previous cursor.  Note that limit used here should
be the same limit as the initial

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
index 14130bc..5dc5502 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResults.java
@@ -49,16 +49,13 @@ public class CandidateResults implements Iterable<CandidateResult>
{
 
     private final List<CandidateResult> candidates;
     private final Collection<SelectFieldMapping> getFieldMappings;
-    private final String esCursor;
 
 
     public CandidateResults( List<CandidateResult> candidates,
-                             final Collection<SelectFieldMapping> getFieldMappings,
-                             final String esCursor
+                             final Collection<SelectFieldMapping> getFieldMappings
     ) {
         this.candidates = candidates;
         this.getFieldMappings = getFieldMappings;
-        this.esCursor = esCursor;
     }
 
 
@@ -113,7 +110,4 @@ public class CandidateResults implements Iterable<CandidateResult>
{
         return candidates.iterator();
     }
 
-    public String getEsCursor() {
-        return esCursor;
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
index 6ba0756..0fce4c8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
@@ -53,6 +53,10 @@ public class SearchTypes {
         return typeNames;
     }
 
+    public String[] getTypes(){
+        return types;
+    }
+
 
     /**
      * Create a search that will search on the specified types

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 5b05b27..6b1f7d5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -26,6 +26,10 @@ import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.ShardOperationFailedException;
@@ -39,6 +43,7 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
+import org.mortbay.util.ajax.JSON;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -141,10 +146,14 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex
{
         return batch;
     }
 
-
     @Override
     public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes,
final String query,
                                     final int limit ) {
+        return search(searchEdge,searchTypes,query,limit,0);
+    }
+
+    public CandidateResults search( final SearchEdge searchEdge, final SearchTypes searchTypes,
final String query,
+                                    final int limit, final int from ) {
 
         IndexValidationUtils.validateSearchEdge( searchEdge );
         Preconditions.checkNotNull( searchTypes, "searchTypes cannot be null" );
@@ -156,7 +165,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex
{
 
         final ParsedQuery parsedQuery = ParsedQueryBuilder.build( query );
 
-        final SearchRequestBuilder srb = searchRequest.getBuilder( searchEdge, searchTypes,
parsedQuery, limit );
+        final SearchRequestBuilder srb = searchRequest.getBuilder( searchEdge, searchTypes,
parsedQuery, limit, from );
 
         if ( logger.isDebugEnabled() ) {
             logger.debug( "Searching index (read alias): {}\n  nodeId: {}, edgeType: {},
 \n type: {}\n   query: {} ",
@@ -177,14 +186,12 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex
{
         }
         failureMonitor.success();
 
-        return parseResults( searchResponse, parsedQuery, limit );
+        return parseResults( searchResponse, searchEdge, searchTypes, parsedQuery, limit,
from );
     }
 
 
     public CandidateResults getNextPage( final String cursor ) {
-        Preconditions.checkNotNull( cursor, "cursor is a required argument" );
-
-        SearchResponse searchResponse;
+        Preconditions.checkNotNull(cursor, "cursor is a required argument");
 
         String userCursorString = cursor;
 
@@ -204,37 +211,9 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex
{
             throw new QueryException( String.format("Could not find a cursor for the value
'%s' ", userCursorString ));
         }
 
-
-
         //parse the query state
         final QueryState queryState = QueryState.fromSerialized( queryStateString );
-
-        //parse the query so we can get select terms
-        final ParsedQuery parsedQuery = ParsedQueryBuilder.build( queryState.ql);
-
-
-        logger.debug( "Executing query with cursor: {} ", queryState.esCursor );
-
-
-        SearchScrollRequestBuilder ssrb =
-                esProvider.getClient().prepareSearchScroll( queryState.esCursor ).setScroll(
cursorTimeout + "m" );
-
-        try {
-            //Added For Graphite Metrics
-            Timer.Context timeSearchCursor = cursorTimer.time();
-            searchResponse = ssrb.execute().actionGet();
-            timeSearchCursor.stop();
-        }
-        catch ( Throwable t ) {
-            logger.error( "Unable to communicate with elasticsearch", t );
-            failureMonitor.fail( "Unable to execute batch", t );
-            throw t;
-        }
-
-
-        failureMonitor.success();
-
-        return parseResults( searchResponse, parsedQuery, queryState.limit );
+        return search(queryState.searchEdge,queryState.searchTypes,queryState.ql,queryState.limit,queryState.from);
     }
 
 
@@ -292,8 +271,9 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex
{
     /**
      * Parse the results and return the canddiate results
      */
-    private CandidateResults parseResults( final SearchResponse searchResponse, final ParsedQuery
query,
-                                           final int limit ) {
+    private CandidateResults parseResults( final SearchResponse searchResponse, final SearchEdge
searchEdge,
+                                           final SearchTypes searchTypes, final ParsedQuery
query,
+                                           final int limit, final int from ) {
 
         final SearchHits searchHits = searchResponse.getHits();
         final SearchHit[] hits = searchHits.getHits();
@@ -310,11 +290,10 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex
{
             candidates.add( candidateResult );
         }
 
-        final String esScrollCursor = searchResponse.getScrollId();
-        final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings(),esScrollCursor);
+        final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings(),from+limit);
 
         // >= seems odd.  However if we get an overflow, we need to account for it.
-        if ( esScrollCursor != null && length >= limit ) {
+        if (  length >= limit ) {
             final String cursor = candidateResults.initializeCursor();
 
             //now set this into our map module
@@ -323,13 +302,12 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex
{
 
             final int storageSeconds = ( int ) TimeUnit.MINUTES.toSeconds( minutes );
 
-            final QueryState state = new QueryState( query.getOriginalQuery(), limit, esScrollCursor
);
+            final QueryState state = new QueryState( query.getOriginalQuery(), searchEdge,
searchTypes, limit, from );
 
             final String queryStateSerialized = state.serialize();
 
-            mapManager.putString (cursor , queryStateSerialized, storageSeconds );
+            mapManager.putString(cursor, queryStateSerialized, storageSeconds);
 
-            logger.debug( " User cursor = {},  Cursor = {} ", cursor, esScrollCursor );
         }
 
         return candidateResults;
@@ -341,6 +319,11 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex
{
      */
     private static final class QueryState implements Serializable{
 
+
+        private static final JsonFactory SMILE_FACTORY = new JsonFactory();
+
+        private static final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
+
         /**
          * Our reserved character for constructing our storage string
          */
@@ -348,15 +331,18 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex
{
 
         private final String ql;
 
+        private final SearchEdge searchEdge;
+        private final SearchTypes searchTypes;
         private final int limit;
+        private final int from;
 
-        private final String esCursor;
 
-
-        private QueryState( final String ql, final int limit, final String esCursor ) {
+        private QueryState( final String ql,final SearchEdge searchEdge, final  SearchTypes
searchTypes, final int limit, final int from ) {
             this.ql = ql;
+            this.searchEdge = searchEdge;
+            this.searchTypes = searchTypes;
             this.limit = limit;
-            this.esCursor = esCursor;
+            this.from = from;
         }
 
 
@@ -365,27 +351,49 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex
{
          */
         public static QueryState fromSerialized( final String input ) {
 
-            final String[] parts = input.split( STORAGE_DELIM );
+            final String[] parts = input.split(STORAGE_DELIM);
 
-            Preconditions.checkArgument( parts != null && parts.length == 3,
-                    "there must be 3 parts to the serialized query state" );
+            Preconditions.checkArgument(parts != null && parts.length == 7,
+                "there must be 5 parts to the serialized query state");
+            try {
 
+                String edgeName = parts[1];
+                Id edgeId = MAPPER.readValue(parts[2],SimpleId.class);
+                SearchEdge.NodeType nodeType = SearchEdge.NodeType.valueOf(parts[3]);
 
-            return new QueryState( parts[0], Integer.parseInt( parts[1] ), parts[2] );
+                String[] searchTypes = MAPPER.readValue(parts[4],String[].class);
+                SearchTypes types = searchTypes.length == 0 ? SearchTypes.allTypes() : SearchTypes.fromTypes(searchTypes);
+                return new QueryState(parts[0], new SearchEdgeImpl(edgeId,edgeName,nodeType),types
, Integer.parseInt(parts[5]), Integer.parseInt(parts[6]));
+            } catch (Exception jpe) {
+                throw new RuntimeException(jpe);
+            }
         }
 
 
         public String serialize() {
+            try {
 
-            StringBuilder storageString = new StringBuilder();
+                StringBuilder storageString = new StringBuilder();
 
-            storageString.append( ql ).append( STORAGE_DELIM );
+                storageString.append(ql).append(STORAGE_DELIM);
 
-            storageString.append( limit ).append( STORAGE_DELIM );
+                String edge =searchEdge.getEdgeName();
+                storageString.append(edge).append(STORAGE_DELIM);
+                String nodeid = MAPPER.writeValueAsString(searchEdge.getNodeId());
+                storageString.append(nodeid).append(STORAGE_DELIM);
+                String nodeType = searchEdge.getNodeType().name();
+                storageString.append(nodeType).append(STORAGE_DELIM);
+                String types = MAPPER.writeValueAsString(searchTypes.getTypes());
+                storageString.append(types).append(STORAGE_DELIM);
 
-            storageString.append( esCursor );
+                storageString.append(limit).append(STORAGE_DELIM);
 
-            return storageString.toString();
+                storageString.append(from+limit);
+
+                return storageString.toString();
+            }catch (Exception jpe){
+                throw new RuntimeException(jpe);
+            }
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
index 31c84de..33a9adf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.persistence.index.impl;
 import java.util.Set;
 
 import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.index.query.BoolFilterBuilder;
@@ -81,14 +82,14 @@ public class SearchRequestBuilderStrategy {
      * Get the search request builder
      */
     public SearchRequestBuilder getBuilder( final SearchEdge searchEdge, final SearchTypes
searchTypes,
-                                            final ParsedQuery query, final int limit ) {
+                                            final ParsedQuery query, final int limit, final
int from) {
 
         Preconditions
                 .checkArgument( limit <= EntityIndex.MAX_LIMIT, "limit is greater than
max " + EntityIndex.MAX_LIMIT );
 
         SearchRequestBuilder srb =
-                esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes( IndexingUtils.ES_ENTITY_TYPE
)
-                          .setScroll( cursorTimeout + "m" );
+                esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes( IndexingUtils.ES_ENTITY_TYPE
).setSearchType(SearchType.QUERY_THEN_FETCH);
+
 
         final QueryVisitor visitor = visitParsedQuery( query );
 
@@ -101,7 +102,7 @@ public class SearchRequestBuilderStrategy {
         srb.setPostFilter( createFilterBuilder( searchEdge, visitor, searchTypes ) );
 
 
-        srb = srb.setFrom( 0 ).setSize( limit );
+        srb = srb.setFrom( from ).setSize( limit );
 
 
         //if we have a geo field, sort by closest to farthest by default

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bbbe9631/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 7112f9e..cd61818 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -642,8 +642,8 @@ public class EntityIndexTest extends BaseIT {
     public void testCursorFormat() throws Exception {
 
         String myType = UUID.randomUUID().toString();
-        Id appId = new SimpleId( "application" );
-        Id ownerId = new SimpleId( "owner" );
+        Id appId = new SimpleId(UUID.randomUUID(), "application" );
+        Id ownerId = new SimpleId( UUID.randomUUID(),"owner" );
 
         ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
 
@@ -701,31 +701,26 @@ public class EntityIndexTest extends BaseIT {
 
         String cursor = null;
         UUID lastId = null;
-
-        for ( int i = 0; i < expectedPages; i++ ) {
+        final String query = "select * where mytype='"+myType+"' order by ordinal asc";
+        int i ;
+        for ( i=0; i < expectedPages; i++ ) {
             //**
-            final String query = "select * where mytype='"+myType+"' order by ordinal asc";
-
-            final CandidateResults results =
-                cursor == null ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query,
limit ) :
-                    entityIndex.getNextPage( cursor );
 
-            assertTrue( results.hasCursor() );
-
-            cursor = results.getCursor();
-
-            assertEquals( "Should be 16 bytes as hex", 32, cursor.length() );
+            final CandidateResults results = cursor == null
+                ? entityIndex.search( indexEdge, SearchTypes.allTypes(), query, limit, i*
limit )
+                : entityIndex.getNextPage(cursor);
 
             assertEquals(limit, results.size());
 
             int ordinal = 0;//i == 0 ? 0 : 1;
             assertNotEquals("Scroll matches last item from previous page",lastId, results.get(ordinal).getId().getUuid());
             lastId = results.get(limit -1).getId().getUuid();
+            cursor = results.getCursor();
             assertEquals("Failed on page "+i ,results.get( ordinal ).getId(), entityIds.get(
i*limit ) );
         }
 
         //get our next page, we shouldn't get a cursor
-        final CandidateResults results = entityIndex.getNextPage( cursor );
+        final CandidateResults results = entityIndex.search(indexEdge, SearchTypes.allTypes(),
query, limit, i * limit);
 
         assertEquals( 0, results.size() );
         assertNull( results.getCursor() );


Mime
View raw message