usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mru...@apache.org
Subject usergrid git commit: Add concurrency to re-index and fix de-serialization problems with Shiro cache, plus comments.
Date Wed, 30 Mar 2016 23:15:06 GMT
Repository: usergrid
Updated Branches:
  refs/heads/release-2.1.1 8014b4f91 -> f5daca59a


Add concurrency to re-index and fix de-serialization problems with Shiro cache, plus comments.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f5daca59
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f5daca59
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f5daca59

Branch: refs/heads/release-2.1.1
Commit: f5daca59a94175a83cb03a7b413eacaa90e78589
Parents: 8014b4f
Author: Michael Russo <mrusso@apigee.com>
Authored: Wed Mar 30 16:14:59 2016 -0700
Committer: Michael Russo <mrusso@apigee.com>
Committed: Wed Mar 30 16:14:59 2016 -0700

----------------------------------------------------------------------
 .../corepersistence/index/IndexProcessorFig.java         | 11 ++++++++++-
 .../corepersistence/index/ReIndexServiceImpl.java        | 10 ++++++----
 .../security/shiro/credentials/AdminUserAccessToken.java |  3 +++
 .../shiro/credentials/ApplicationClientCredentials.java  |  5 +++++
 .../security/shiro/principals/AdminUserPrincipal.java    |  3 +++
 .../shiro/principals/ApplicationGuestPrincipal.java      |  4 +++-
 .../security/shiro/principals/ApplicationPrincipal.java  |  4 +++-
 .../shiro/principals/ApplicationUserPrincipal.java       |  3 +++
 .../security/shiro/principals/OrganizationPrincipal.java |  4 +++-
 9 files changed, 39 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index 15f9f9d..b94da65 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -44,6 +44,8 @@ public interface IndexProcessorFig extends GuicyFig {
 
     String REINDEX_BUFFER_SIZE = "elasticsearch.reindex.buffer_size";
 
+    String REINDEX_CONCURRENCY_FACTOR = "elasticsearch.reindex.concurrency.factor";
+
 
     /**
      * Set the amount of time to wait when Elasticsearch rejects a requests before
@@ -88,11 +90,18 @@ public interface IndexProcessorFig extends GuicyFig {
     @Key(ELASTICSEARCH_QUEUE_IMPL)
     String getQueueImplementation();
 
-    @Default("1000")
+    @Default("100")
     @Key(REINDEX_BUFFER_SIZE)
     int getReindexBufferSize();
 
     /**
+     * The number of parallel buffers during re-index that can be processed
+     */
+    @Default("10")
+    @Key(REINDEX_CONCURRENCY_FACTOR)
+    int getReindexConcurrencyFactor();
+
+    /**
      * Flag to resolve the LOCAL queue implementation service synchronously.
      */
     @Default("false")

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index da2ef1d..b5f7d77 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -125,12 +125,14 @@ public class ReIndexServiceImpl implements ReIndexService {
         final Observable<List<EdgeScope>> runningReIndex = allEntityIdsObservable.getEdgesToEntities(
applicationScopes,
             reIndexRequestBuilder.getCollectionName(), cursorSeek.getSeekValue() )
             .buffer( indexProcessorFig.getReindexBufferSize())
-            .doOnNext(edges -> {
+            .flatMap( edgeScopes -> Observable.just(edgeScopes)
+                .doOnNext(edges -> {
 
-                logger.info("Sending batch of {} to be indexed.", edges.size());
-                indexService.indexBatch(edges, modifiedSince);
+                    logger.info("Sending batch of {} to be indexed.", edges.size());
+                    indexService.indexBatch(edges, modifiedSince);
+                })
+                .subscribeOn( Schedulers.io() ), indexProcessorFig.getReindexConcurrencyFactor());
 
-            });
 
 
         //start our sampler and state persistence

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java
b/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java
index a2f3743..4c3ffce 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java
@@ -19,6 +19,9 @@ package org.apache.usergrid.security.shiro.credentials;
 
 public class AdminUserAccessToken extends AbstractAccessTokenCredentials implements AdminUserCredentials
{
 
+    /**
+     * Needed for Jackson. Do not remove
+     */
     public AdminUserAccessToken() {}
 
     public AdminUserAccessToken( String token ) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java
b/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java
index 8c20bb6..0972c37 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java
@@ -19,6 +19,11 @@ package org.apache.usergrid.security.shiro.credentials;
 
 public class ApplicationClientCredentials extends AbstractClientCredentials implements ApplicationCredentials
{
 
+    /**
+     * Needed for Jackson, do not remove
+     */
+    public ApplicationClientCredentials(){}
+
     public ApplicationClientCredentials( String key, String secret ) {
         super( key, secret );
     }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java
b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java
index 0e18f35..a594d1e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java
@@ -37,6 +37,9 @@ import static org.apache.usergrid.security.shiro.utils.SubjectUtils.getPermissio
 public class AdminUserPrincipal extends UserPrincipal {
     private static final Logger logger = LoggerFactory.getLogger(AdminUserPrincipal.class);
 
+    /**
+     * Needed for Jackson, do not remove
+     */
     public AdminUserPrincipal() {
     }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java
b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java
index 38da94d..9e9d821 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java
@@ -43,7 +43,9 @@ public class ApplicationGuestPrincipal extends PrincipalIdentifier {
 
     ApplicationInfo application;
 
-
+    /**
+     * Needed for Jackson, do not remove
+     */
     public ApplicationGuestPrincipal( ) {}
 
     public ApplicationGuestPrincipal( ApplicationInfo application ) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java
b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java
index 5252052..cff9daa 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java
@@ -34,7 +34,9 @@ public class ApplicationPrincipal extends PrincipalIdentifier {
 
     ApplicationInfo application;
 
-
+    /**
+     * Needed for Jackson, do not remove
+     */
     public ApplicationPrincipal() {}
 
     public ApplicationPrincipal( ApplicationInfo application ) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java
b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java
index 962fcf5..65503b4 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java
@@ -45,6 +45,9 @@ public class ApplicationUserPrincipal extends UserPrincipal {
 
     private static final Logger logger = LoggerFactory.getLogger(AdminUserPrincipal.class);
 
+    /**
+     * Needed for Jackson, do not remove
+     */
     public ApplicationUserPrincipal() {}
 
     public ApplicationUserPrincipal( UUID applicationId, UserInfo user ) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java
b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java
index e885365..0ebbaa6 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java
@@ -40,7 +40,9 @@ public class OrganizationPrincipal extends PrincipalIdentifier {
 
     OrganizationInfo organization;
 
-
+    /**
+     * Needed for Jackson, do not remove
+     */
     public OrganizationPrincipal() {
     }
 


Mime
View raw message