usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sfeld...@apache.org
Subject [3/5] incubator-usergrid git commit: single index
Date Wed, 18 Mar 2015 21:59:54 GMT
single index


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/346eba98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/346eba98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/346eba98

Branch: refs/heads/USERGRID-480
Commit: 346eba98ed1080940730fb617951e6172a1683bb
Parents: 1199e47
Author: Shawn Feldman <sfeldman@apache.org>
Authored: Wed Mar 18 14:56:36 2015 -0600
Committer: Shawn Feldman <sfeldman@apache.org>
Committed: Wed Mar 18 14:56:36 2015 -0600

----------------------------------------------------------------------
 .../corepersistence/CpEntityManagerFactory.java |  3 -
 .../usergrid/persistence/index/EntityIndex.java | 21 ------
 .../persistence/index/IndexIdentifier.java      | 16 +----
 .../usergrid/persistence/index/SearchTypes.java | 14 +++-
 .../index/impl/EsEntityIndexBatchImpl.java      |  6 +-
 .../index/impl/EsEntityIndexImpl.java           | 69 +++++---------------
 .../persistence/index/impl/IndexingUtils.java   |  8 ++-
 .../persistence/index/impl/EntityIndexTest.java | 65 +-----------------
 8 files changed, 42 insertions(+), 160 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index fe4d828..2bdb54c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -292,7 +292,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
         EntityManager appEm = getEntityManager( applicationId );
 
         appEm.create( applicationId, TYPE_APPLICATION, properties );
-        appEm.createIndex();
         appEm.resetRoles();
         appEm.refreshIndex();
 
@@ -706,7 +705,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
         EntityManager em = getEntityManager( appId );
 
         //explicitly invoke create index, we don't know if it exists or not in ES during
a rebuild.
-        em.createIndex();
         Application app = em.getApplication();
 
         em.reindex( po );
@@ -760,7 +758,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
         EntityManager em = getEntityManager( appId );
 
         //explicitly invoke create index, we don't know if it exists or not in ES during
a rebuild.
-        em.createIndex();
         Application app = em.getApplication();
 
         em.reindexCollection( po, collectionName, reverse );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index aa2bf8e..c8840fa 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -68,27 +68,6 @@ public interface EntityIndex {
     public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType,
Query query );
 
     /**
-     * Get the candidate results of all versions of the entity for this id.
-     * @param indexScope The scope of the index to search in
-     * @param id The id to search within.
-     */
-    public CandidateResults getEntityVersions(final IndexScope indexScope, final Id id);
-
-//    /**
-//     * Create a delete method that deletes by Id. This will delete all documents from ES
with the same entity Id,
-//     * effectively removing all versions of an entity from all index scopes
-//     * @param entityId The entityId to remove
-//     */
-//    public Future deleteAllVersionsOfEntity(final Id entityId );
-//
-//    /**
-//     * Takes all the previous versions of the current entity and deletes all previous versions
-//     * @param id The id to remove
-//     * @param version The max version to retain
-//     */
-//    public Future deletePreviousVersions(final Id id, final UUID version);
-
-    /**
      * Refresh the index.
      */
     public void refresh();

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index f23dc36..8465d84 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -40,7 +40,7 @@ public class IndexIdentifier{
      * @return
      */
     public IndexAlias getAlias() {
-        return new IndexAlias(config,getIndexBase());
+        return new IndexAlias(config,config.getIndexPrefix());
     }
 
     /**
@@ -50,22 +50,12 @@ public class IndexIdentifier{
      */
     public String getIndex(String suffix) {
         if (suffix != null) {
-            return getIndexBase() + "_" + suffix;
+            return config.getIndexPrefix() + "_" + suffix;
         } else {
-            return getIndexBase();
+            return config.getIndexPrefix();
         }
     }
 
-    /**
-     * returns the base name for index which will be used to add an alias and index
-     * @return
-     */
-    private String getIndexBase() {
-        StringBuilder sb = new StringBuilder();
-        sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR);
-        IndexingUtils.idString(sb, applicationScope.getApplication());
-        return sb.toString();
-    }
 
     public class IndexAlias{
         private final String readAlias;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
index 35b3a8b..1a97da2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
@@ -20,6 +20,10 @@ package org.apache.usergrid.persistence.index;/*
  */
 
 
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.impl.IndexingUtils;
+import org.apache.usergrid.persistence.model.entity.Id;
+
 import java.util.Arrays;
 
 
@@ -37,11 +41,17 @@ public class SearchTypes {
     private SearchTypes( final String... types ) {this.types = types;}
 
 
-    public String[] getTypeNames() {
-        return types;
+    public String[] getTypeNames(ApplicationScope applicationScope) {
+        String[] typeNames = new String[types.length];
+        int i =0 ;
+        for(String type : types){
+            typeNames[i++] = IndexingUtils.getType(applicationScope,type);
+        }
+        return typeNames;
     }
 
 
+
     /**
      * Create a search that will search on the specified types
      * @param types

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 23b9041..977fe51 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -117,9 +117,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
         String indexId = createIndexDocId( entity, context );
 
         log.debug( "Indexing entity documentId {} data {} ", indexId, entityAsMap );
-        final String entityType = entity.getId().getType();
-
-
+        final String entityType =IndexingUtils.getType(applicationScope, entity.getId());
         container.addIndexRequest(new IndexRequest(alias.getWriteAlias(), entityType, indexId,
entityAsMap));
 
         return this;
@@ -134,7 +132,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
         ValidationUtils.verifyVersion( version );
 
         final String context = createContextName(indexScope);
-        final String entityType = id.getType();
+        final String entityType =IndexingUtils.getType(applicationScope, id);
 
         final String indexId = createIndexDocId( id, version, context );
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 598c8d9..110b320 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -199,9 +199,9 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
         final int numberOfShards = config.getNumberOfShards();
         final int numberOfReplicas = config.getNumberOfReplicas();
         String[] indexes = getIndexesFromEs(AliasType.Write);
-        if(indexes == null || indexes.length==0) {
+//        if(indexes == null || indexes.length==0) {
             addIndex(null, numberOfShards, numberOfReplicas, config.getWriteConsistencyLevel());
-        }
+//        }
     }
 
     @Override
@@ -233,21 +233,21 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
 
                 //ONLY add the alias if we create the index, otherwise we're going to overwrite
production settings
 
-                /**
-                 * DO NOT MOVE THIS LINE OF CODE UNLESS YOU REALLY KNOW WHAT YOU'RE DOING!!!!
-                 */
-
-                //We do NOT want to create an alias if the index already exists, we'll overwrite
the indexes that
-                //may have been set via other administrative endpoint
-
-                addAlias(normalizedSuffix);
-
-                testNewIndex();
 
                 logger.info("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged());
             } catch (IndexAlreadyExistsException e) {
                 logger.info("Index Name [{}] already exists", indexName);
             }
+            /**
+             * DO NOT MOVE THIS LINE OF CODE UNLESS YOU REALLY KNOW WHAT YOU'RE DOING!!!!
+             */
+
+            //We do NOT want to create an alias if the index already exists, we'll overwrite
the indexes that
+            //may have been set via other administrative endpoint
+
+            addAlias(normalizedSuffix);
+
+            testNewIndex();
 
 
 
@@ -347,7 +347,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
         // to receive documents. Occasionally we see errors.
         // See this post: http://s.apache.org/index-missing-exception
 
-        logger.debug( "Testing new index name: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
+        logger.debug("Testing new index name: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
 
         final RetryOperation retryOperation = new RetryOperation() {
             @Override
@@ -410,12 +410,9 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
     public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes,
             final Query query ) {
 
-        final String context = IndexingUtils.createContextName( indexScope );
-        final String[] entityTypes = searchTypes.getTypeNames();
-
+        final String context = IndexingUtils.createContextName(indexScope);
+        final String[] entityTypes = searchTypes.getTypeNames(applicationScope);
         QueryBuilder qb = query.createQueryBuilder(context);
-
-
         SearchResponse searchResponse;
 
         if ( query.getCursor() == null ) {
@@ -635,42 +632,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
     }
 
 
-    @Override
-    public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
-
-        //since we don't have paging inputs, there's no point in executing a query for paging.
-
-        final String context = IndexingUtils.createContextName(scope);
-        final SearchTypes searchTypes = SearchTypes.fromTypes(id.getType());
-
-        final QueryBuilder queryBuilder =
-                QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context
);
-
-        final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias()
)
-                .setTypes(searchTypes.getTypeNames())
-                .setScroll(cursorTimeout + "m")
-                .setQuery(queryBuilder);
-
-        final SearchResponse searchResponse;
-        try {
-            //Added For Graphite Metrics
-            Timer.Context timeEntityIndex = getVersionsTimer.time();
-            searchResponse = srb.execute().actionGet();
-            timeEntityIndex.stop();
-        }
-        catch ( Throwable t ) {
-            logger.error( "Unable to communicate with elasticsearch" );
-            failureMonitor.fail( "Unable to execute batch", t);
-            throw t;
-        }
-
-
-        failureMonitor.success();
-
-        return parseResults(searchResponse, new Query());
-    }
-
-
 
     /**
      * Completely delete an index.

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 0df3a2c..aa6d0f3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -230,7 +230,7 @@ public class IndexingUtils {
                             .startObject("template__long")
                                 .field("match", LONG_PREFIX + "*")
                                 .field("match_mapping_type", "long")
-                                .startObject("mapping").field("type", "long").field("index","not_analyzed").field(DOC_VALUES_KEY,
true).endObject()
+                                .startObject("mapping").field("type", "long").field("index",
"not_analyzed").field(DOC_VALUES_KEY, true).endObject()
                             .endObject()
                         .endObject()
 
@@ -249,5 +249,11 @@ public class IndexingUtils {
     }
 
 
+    public static String getType(ApplicationScope applicationScope, Id entityId) {
+        return getType(applicationScope,entityId.getType());
+    }
 
+    public static String getType(ApplicationScope applicationScope, String type) {
+        return idString(applicationScope.getApplication()) + SEPARATOR + type;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index ca9bf79..22ca51f 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -271,7 +271,7 @@ public class EntityIndexTest extends BaseIT {
         ObjectMapper mapper = new ObjectMapper();
         List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>()
{} );
         EntityIndexBatch batch = entityIndex.createBatch();
-        insertJsonBlob(sampleJson,batch, entityType, indexScope, max,startIndex);
+        insertJsonBlob(sampleJson,batch, entityType, indexScope, max, startIndex);
         batch.execute().get();
         entityIndex.refresh();
     }
@@ -297,10 +297,9 @@ public class EntityIndexTest extends BaseIT {
             EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
             entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
             batch.index(indexScope, entity);
+            batch.execute().get();
+
 
-            if(count %1000 == 0){
-                batch.execute().get();
-            }
             if ( ++count > max ) {
                 break;
             }
@@ -370,7 +369,6 @@ public class EntityIndexTest extends BaseIT {
 
     private void testQueries(final IndexScope scope, SearchTypes searchTypes, final EntityIndex
entityIndex ) {
 
-
         testQuery(scope, searchTypes, entityIndex, "name = 'Morgan Pierce'", 1 );
 
         testQuery(scope, searchTypes, entityIndex, "name = 'morgan pierce'", 1 );
@@ -443,63 +441,6 @@ public class EntityIndexTest extends BaseIT {
         }
     }
 
-
-    @Test
-    public void getEntityVersions() throws Exception {
-
-        Id appId = new SimpleId( "application" );
-        Id ownerId = new SimpleId( "owner" );
-
-        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-
-        IndexScope indexScope = new IndexScopeImpl( ownerId, "users" );
-
-
-
-        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
-        entityIndex.initializeIndex();
-
-        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
-        Map<String, Object> properties = new LinkedHashMap<String, Object>();
-        properties.put( "username", "edanuff" );
-        properties.put( "email", "ed@anuff.com" );
-        properties.put( "middlename", middleName );
-
-        Map entityMap = new HashMap() {{
-            put( "username", "edanuff" );
-            put( "email", "ed@anuff.com" );
-            put( "middlename", middleName );
-        }};
-
-        final Id userId = new SimpleId("user");
-
-        Entity user = EntityIndexMapUtils.fromMap( entityMap );
-        EntityUtils.setId( user, userId);
-        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
-
-
-        final EntityIndexBatch batch = entityIndex.createBatch();
-        user.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
-
-        batch.index( indexScope, user );
-
-        user.setField( new StringField( "address1", "1782 address st" ) );
-        batch.index( indexScope, user );
-        user.setField( new StringField( "address2", "apt 508" ) );
-        batch.index( indexScope,  user );
-        user.setField( new StringField( "address3", "apt 508" ) );
-        batch.index( indexScope,  user);
-        batch.execute().get();
-        entityIndex.refresh();
-
-        CandidateResults results = entityIndex.getEntityVersions(indexScope,  user.getId()
);
-
-        assertEquals(1,  results.size());
-        assertEquals( results.get( 0 ).getId(), user.getId() );
-        assertEquals( results.get(0).getVersion(), user.getVersion());
-    }
-
-
     @Test
     public void deleteVerification() throws Throwable {
 


Mime
View raw message