usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mru...@apache.org
Subject [18/26] usergrid git commit: Made max-retries and intervals configurable for both lock manager and entity manager factory, also better error logging.
Date Fri, 03 Jun 2016 14:46:08 GMT
Made max-retries and intervals configurable for both lock manager and entity manager factory,
also better error logging.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/61a35a04
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/61a35a04
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/61a35a04

Branch: refs/heads/release-2.1.1
Commit: 61a35a041598af0e336199343ed6a489454980f6
Parents: fcd00e8
Author: Dave Johnson <snoopdave@apache.org>
Authored: Thu May 26 11:10:33 2016 -0400
Committer: Dave Johnson <snoopdave@apache.org>
Committed: Thu May 26 11:10:33 2016 -0400

----------------------------------------------------------------------
 .../corepersistence/CpEntityManagerFactory.java | 83 ++++++++++++++++----
 .../cassandra/AstyanaxLockManagerImpl.java      | 17 +++-
 .../exception/CollectionRuntimeException.java   | 11 +++
 3 files changed, 93 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/61a35a04/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index b6fbc2a..ee28765 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -20,11 +20,10 @@ import com.google.common.base.Optional;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.google.inject.Injector;
 import com.google.inject.Key;
 import com.google.inject.TypeLiteral;
-import com.netflix.astyanax.connectionpool.exceptions.BadRequestException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
 import org.apache.commons.lang.StringUtils;
 import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
 import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
@@ -39,6 +38,7 @@ import org.apache.usergrid.persistence.cassandra.CassandraService;
 import org.apache.usergrid.persistence.cassandra.CounterUtils;
 import org.apache.usergrid.persistence.cassandra.Setup;
 import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
 import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
 import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
 import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
@@ -68,6 +68,7 @@ import java.util.*;
 import static java.lang.String.CASE_INSENSITIVE_ORDER;
 import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
 import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
+import static org.apache.usergrid.persistence.Schema.initLock;
 
 
 /**
@@ -107,7 +108,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
     private final GraphManagerFactory graphManagerFactory;
     private final IndexSchemaCacheFactory indexSchemaCacheFactory;
 
-    public static final String MANAGEMENT_APP_MAX_RETRIES= "management.app.max.retries";
+    public static final String MANAGEMENT_APP_INIT_MAXRETRIES= "management.app.init.max-retries";
+    public static final String MANAGEMENT_APP_INIT_INTERVAL = "management.app.init.interval";
 
 
     public CpEntityManagerFactory(
@@ -177,12 +179,19 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
                         }
                     }
 
-                    if ( app == null && throwable != null && throwable.getCause()
instanceof BadRequestException) {
-                        // probably means schema has not been created yet
+                    final boolean missingKeyspace;
+                    if ( throwable instanceof CollectionRuntimeException ) {
+                        CollectionRuntimeException cre = (CollectionRuntimeException) throwable;
+                        missingKeyspace = cre.isMissingKeyspace();
                     } else {
-                        throw new RuntimeException( "Error getting application " + appId,
throwable );
+                        missingKeyspace = false;
                     }
 
+                    if ( app == null && !missingKeyspace ) {
+                        throw new RuntimeException( "Error getting application " + appId,
throwable );
+
+                    } // else keyspace is missing because setup/bootstrap not done yet
+
                     return entityManager;
                 }
             });
@@ -190,34 +199,77 @@ public class CpEntityManagerFactory implements EntityManagerFactory,
Application
 
 
     private void checkManagementApp(Properties properties) {
+
         int maxRetries = 100;
         try {
-            maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_MAX_RETRIES,
"100" ));
+            maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_INIT_MAXRETRIES,
"100" ));
+
+        } catch ( Exception e ) {
+            logger.error("Error parsing " + MANAGEMENT_APP_INIT_MAXRETRIES + ". Will use
" + maxRetries, e );
+        }
+
+        int interval = 1000;
+        try {
+            interval = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_INIT_INTERVAL,
"1000" ));
 
         } catch ( Exception e ) {
-            logger.error("Error parsing " + MANAGEMENT_APP_MAX_RETRIES + ". Will use " +
maxRetries, e );
+            logger.error("Error parsing " + MANAGEMENT_APP_INIT_INTERVAL + ". Will use "
+ maxRetries, e );
         }
 
         // hold up construction until we can access the management app
         int retries = 0;
         boolean managementAppFound = false;
+        boolean bootstrapping = false;
+        Set<Class> seenBefore = new HashSet<>(10);
         while ( !managementAppFound && retries++ < maxRetries ) {
             try {
-                getEntityManager( getManagementAppId() ).getApplication();
+                // bypass entity manager cache and get managementApp
+                managementApp = _getEntityManager( getManagementAppId() ).getApplication();
                 managementAppFound = true;
 
             } catch ( Throwable t ) {
-                String msg = "Error " + t.getClass() + " getting management app on try "
+ retries;
-                if ( logger.isDebugEnabled() ) {
+
+                if ( t instanceof CollectionRuntimeException ) {
+                    CollectionRuntimeException cre = (CollectionRuntimeException)t;
+                    if ( cre.isMissingKeyspace() ) {
+                        // we're bootstrapping, ignore this and continue
+                        bootstrapping = true;
+                        break;
+                    }
+                }
+                Throwable cause = t;
+
+                // there was an error, be as informative as possible
+                StringBuilder sb = new StringBuilder();
+                sb.append(retries).append(": Error (");
+
+                if ( t instanceof UncheckedExecutionException ) {
+                    UncheckedExecutionException uee = (UncheckedExecutionException)t;
+                    if ( uee.getCause() instanceof RuntimeException ) {
+                        cause = uee.getCause().getCause();
+                        sb.append(cause.getClass().getSimpleName()).append(") ")
+                          .append(uee.getCause().getMessage());
+                    } else {
+                        cause = uee.getCause();
+                        sb.append(cause.getClass().getSimpleName()).append(") ").append(t.getMessage());
+                    }
+                } else {
+                    sb.append(t.getCause().getClass().getSimpleName()).append(") ").append(t.getMessage());
+                }
+
+                String msg = sb.toString();
+                if ( !seenBefore.contains( cause.getClass() ) ) {
                     logger.error( msg, t);
                 } else {
                     logger.error(msg);
                 }
-                try { Thread.sleep( 1000 ); } catch (InterruptedException ignored) {}
+                seenBefore.add( cause.getClass() );
+
+                try { Thread.sleep( interval ); } catch (InterruptedException ignored) {}
             }
         }
 
-        if ( !managementAppFound ) {
+        if ( !managementAppFound && !bootstrapping ) {
             // exception here will prevent WAR from being deployed
             throw new RuntimeException( "Unable to get management app after " + retries +
" retries" );
         }
@@ -323,8 +375,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
         // Ensure the management application is initialized
         initMgmtAppInternal();
 
-        EntityManager managementEm = getEntityManager( getManagementAppId() );
-        EntityManager appEm = getEntityManager(applicationId);
+        // Get entity managers by bypassing the entity manager cache because it expects apps
to already exist
+        final EntityManager managementEm = _getEntityManager( getManagementAppId() );
+        EntityManager appEm = _getEntityManager(applicationId);
 
         final String appName = buildAppName(organizationName, name);
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/61a35a04/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
index 9bf6694..6acce47 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
@@ -22,6 +22,8 @@ import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.netflix.astyanax.Keyspace;
 import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.connectionpool.exceptions.NoAvailableHostsException;
+import com.netflix.astyanax.connectionpool.exceptions.PoolTimeoutException;
 import com.netflix.astyanax.ddl.ColumnFamilyDefinition;
 import com.netflix.astyanax.ddl.KeyspaceDefinition;
 import com.netflix.astyanax.model.ColumnFamily;
@@ -63,6 +65,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
         int maxRetries = cassandraFig.getLockManagerInitRetries();
         int retries = 0;
         boolean famReady = false;
+        Set<Class> seenBefore = new HashSet<>(10);
         while ( !famReady && retries++ < maxRetries ) {
             try {
                 keyspace = cassandraCluster.getLocksKeyspace();
@@ -71,13 +74,21 @@ public class AstyanaxLockManagerImpl implements LockManager {
                 famReady = true;
 
             } catch ( Throwable t ) {
-                String msg = "Error " + t.getClass().getSimpleName() + " creating locks keyspace
try " + retries;
-                if ( logger.isDebugEnabled() ) {
+                final String msg;
+                if ( t instanceof PoolTimeoutException || t instanceof NoAvailableHostsException)
{
+                    msg = retries + ": Cannot connect to Cassandra (" + t.getClass().getSimpleName()
+ ")";
+                } else {
+                    msg = retries + ": Error (" + t.getClass().getSimpleName() + ") tries="
+ retries;
+                }
+                if ( !seenBefore.contains( t.getClass() ) ) {
                     logger.error( msg, t );
                 } else {
                     logger.error( msg );
                 }
-                try { Thread.sleep( cassandraFig.getLockManagerInitInterval() ); } catch
(InterruptedException ignored) {}
+                seenBefore.add( t.getClass() );
+                try {
+                    Thread.sleep( cassandraFig.getLockManagerInitInterval() );
+                } catch (InterruptedException ignored) {}
             }
         }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/61a35a04/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
index 431122e..f27e2d7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
@@ -17,6 +17,7 @@
  */
 package org.apache.usergrid.persistence.collection.exception;
 
+import com.netflix.astyanax.connectionpool.exceptions.BadRequestException;
 import org.apache.usergrid.persistence.collection.MvccEntity;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 
@@ -56,6 +57,16 @@ public class CollectionRuntimeException extends RuntimeException {
         this.applicationScope = scope;
     }
 
+    public boolean isMissingKeyspace() {
+        if ( getCause() instanceof BadRequestException ) {
+            BadRequestException bre = (BadRequestException)getCause();
+            String msg = bre.getMessage();
+            if ( msg.contains("Keyspace") && msg.contains( "does not exist" ) ) {
+                return true;
+            }
+        }
+        return false;
+    }
 
     public ApplicationScope getApplicationScope() {
         return applicationScope;


Mime
View raw message