usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From toddn...@apache.org
Subject [01/14] incubator-usergrid git commit: fix deletes so we remove from all
Date Wed, 10 Dec 2014 00:31:41 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/USERGRID-252 5d4ce7261 -> 43ed8464f


fix deletes so we remove from all


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4b90e97e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4b90e97e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4b90e97e

Branch: refs/heads/USERGRID-252
Commit: 4b90e97e16253559d51479dd57a59aacf3f61466
Parents: c4973e9
Author: Shawn Feldman <sfeldman@apache.org>
Authored: Mon Dec 8 08:40:54 2014 -0700
Committer: Shawn Feldman <sfeldman@apache.org>
Committed: Mon Dec 8 08:40:54 2014 -0700

----------------------------------------------------------------------
 .../usergrid/persistence/index/EntityIndex.java |  7 ++++
 .../index/impl/EsEntityIndexBatchImpl.java      | 24 ++++++++----
 .../index/impl/EsEntityIndexImpl.java           | 18 ++++++---
 .../persistence/index/impl/EntityIndexTest.java | 41 +++++++++++++++++++-
 4 files changed, 76 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b90e97e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 4a653b3..6e7779f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -47,6 +47,13 @@ public interface EntityIndex {
     public void addIndex(final String indexSuffix, final int shards, final int replicas);
 
     /**
+     * Get the indexes for an alias
+     * @param aliasName name of alias
+     * @return list of index names
+     */
+    public String[] getIndexes(final String aliasName);
+
+    /**
      * Create the index batch.
      */
     public EntityIndexBatch createBatch();

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b90e97e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index a3ca8a5..35c89ae 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -26,7 +26,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 
-import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.*;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -36,9 +36,6 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.IndexScope;
 import org.apache.usergrid.persistence.index.query.CandidateResult;
 import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
 import org.apache.usergrid.persistence.model.entity.Entity;
@@ -59,6 +56,9 @@ import org.apache.usergrid.persistence.model.field.UUIDField;
 import org.apache.usergrid.persistence.model.field.value.EntityObject;
 
 import com.google.common.base.Joiner;
+import rx.Observable;
+import rx.functions.Action1;
+import rx.schedulers.Schedulers;
 
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
@@ -92,13 +92,16 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
     private final FailureMonitor failureMonitor;
 
+    private final EntityIndex entityIndex;
+
 
     public EsEntityIndexBatchImpl( final ApplicationScope applicationScope, final Client
client, 
-            final IndexFig config, final int autoFlushSize, final FailureMonitor failureMonitor
) {
+            final IndexFig config, final int autoFlushSize, final FailureMonitor failureMonitor,
final EntityIndex entityIndex ) {
 
         this.applicationScope = applicationScope;
         this.client = client;
         this.failureMonitor = failureMonitor;
+        this.entityIndex = entityIndex;
         this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, applicationScope);
         this.alias = indexIdentifier.getAlias();
         this.refresh = config.isForcedRefresh();
@@ -162,7 +165,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
         final String context = createContextName( indexScope );
         final String entityType = id.getType();
 
-        String indexId = createIndexDocId( id, version, context );
+        final String indexId = createIndexDocId( id, version, context );
 
 
         if ( log.isDebugEnabled() ) {
@@ -181,8 +184,15 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
 
         log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
+        final String[] indexes = entityIndex.getIndexes(alias.getReadAlias());
+        //get all indexes then flush everyone
+        Observable.from(indexes).subscribeOn(Schedulers.io()).forEach(new Action1<String>()
{
+            @Override
+            public void call(String index) {
+                bulkRequest.add(client.prepareDelete(index, entityType, indexId).setRefresh(refresh));
 
-        bulkRequest.add( client.prepareDelete(alias.getWriteAlias(), entityType, indexId
).setRefresh( refresh ) );
+            }
+        });
 
         log.debug( "Deindexed Entity with index id " + indexId );
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b90e97e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index d546092..13d07d6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -178,9 +178,9 @@ public class EsEntityIndexImpl implements EntityIndex {
             Boolean isAck;
             String indexName = indexIdentifier.getIndex(indexSuffix);
             final AdminClient adminClient = esProvider.getClient().admin();
-            //remove write alias, can only have one
-            ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new
GetAliasesRequest(alias.getWriteAlias())).actionGet().getAliases();
-            String[] indexNames = aliasMap.keys().toArray(String.class);
+            final String aliasName = alias.getWriteAlias();
+            String[] indexNames = getIndexes( aliasName);
+
             for(String currentIndex : indexNames){
                 isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,alias.getWriteAlias()).execute().actionGet().isAcknowledged();
                 logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]",currentIndex,
alias, isAck);
@@ -198,6 +198,14 @@ public class EsEntityIndexImpl implements EntityIndex {
         }
     }
 
+    @Override
+    public String[] getIndexes(String aliasName) {
+        final AdminClient adminClient = esProvider.getClient().admin();
+        //remove write alias, can only have one
+        ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new
GetAliasesRequest(aliasName)).actionGet().getAliases();
+        return aliasMap.keys().toArray(String.class);
+    }
+
 
     /**
      * Tests writing a document to a new index to ensure it's working correctly. See this
post:
@@ -262,7 +270,7 @@ public class EsEntityIndexImpl implements EntityIndex {
     @Override
     public EntityIndexBatch createBatch() {
         return new EsEntityIndexBatchImpl( 
-                applicationScope, esProvider.getClient(), config, 1000, failureMonitor );
+                applicationScope, esProvider.getClient(), config, 1000, failureMonitor, this
);
     }
 
 
@@ -424,7 +432,7 @@ public class EsEntityIndexImpl implements EntityIndex {
             @Override
             public boolean doOp() {
                 try {
-                    esProvider.getClient().admin().indices().prepareRefresh( alias.getWriteAlias()
).execute().actionGet();
+                    esProvider.getClient().admin().indices().prepareRefresh( alias.getReadAlias()
).execute().actionGet();
                     logger.debug( "Refreshed index: " + alias);
 
                     return true;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4b90e97e/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 13d1552..ced682b 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -25,6 +25,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.usergrid.persistence.index.query.CandidateResult;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.slf4j.Logger;
@@ -139,6 +140,41 @@ public class EntityIndexTest extends BaseIT {
         testQuery(indexScope, searchTypes, entityIndex, "name = 'Lowe Kelley'", 1 );
     }
 
+    @Test
+    public void testDeleteByQueryWithAlias() throws IOException {
+        Id appId = new SimpleId( "application" );
+
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        EntityIndex entityIndex = eif.createEntityIndex(applicationScope);
+        entityIndex.initializeIndex();
+
+        final String entityType = "thing";
+        IndexScope indexScope = new IndexScopeImpl( appId, "things" );
+        final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+
+        insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",1,0);
+
+        entityIndex.refresh();
+
+        entityIndex.addIndex("v2", 1,0);
+
+        insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",1,0);
+
+        entityIndex.refresh();
+        CandidateResults crs = testQuery(indexScope, searchTypes, entityIndex, "name = 'Bowers
Oneil'", 2);
+
+        EntityIndexBatch entityIndexBatch = entityIndex.createBatch();
+        entityIndexBatch.deindex(indexScope, crs.get(0));
+        entityIndexBatch.deindex(indexScope,crs.get(1));
+        entityIndexBatch.executeAndRefresh();
+        entityIndex.refresh();
+
+        //Hilda Youn
+        testQuery(indexScope, searchTypes, entityIndex, "name = 'Bowers Oneil'", 0);
+
+    }
+
     private void insertJsonBlob(EntityIndex entityIndex, String entityType, IndexScope indexScope,
String filePath,final int max,final int startIndex) throws IOException {
         InputStream is = this.getClass().getResourceAsStream( filePath );
         ObjectMapper mapper = new ObjectMapper();
@@ -172,7 +208,7 @@ public class EntityIndexTest extends BaseIT {
 
 
 
-            if ( count++ > max ) {
+            if ( ++count > max ) {
                 break;
             }
         }
@@ -221,7 +257,7 @@ public class EntityIndexTest extends BaseIT {
     }
 
 
-    private void testQuery(final IndexScope scope, final SearchTypes searchTypes, final EntityIndex
entityIndex, final String queryString, final int num ) {
+    private CandidateResults testQuery(final IndexScope scope, final SearchTypes searchTypes,
final EntityIndex entityIndex, final String queryString, final int num ) {
 
         StopWatch timer = new StopWatch();
         timer.start();
@@ -232,6 +268,7 @@ public class EntityIndexTest extends BaseIT {
 
         assertEquals( num, candidateResults.size() );
         log.debug( "Query time {}ms", timer.getTime() );
+        return candidateResults;
     }
 
 


Mime
View raw message