usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mru...@apache.org
Subject usergrid git commit: Introduce an in-memory cache for Shiro.
Date Fri, 01 Apr 2016 05:05:32 GMT
Repository: usergrid
Updated Branches:
  refs/heads/release-2.1.1 f5daca59a -> 9b56fb2ce


Introduce an in-memory cache for Shiro.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9b56fb2c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9b56fb2c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9b56fb2c

Branch: refs/heads/release-2.1.1
Commit: 9b56fb2ce2bcf5f2da68e9ba9b1f5e9d8feca95e
Parents: f5daca5
Author: Michael Russo <mrusso@apigee.com>
Authored: Thu Mar 31 22:05:27 2016 -0700
Committer: Michael Russo <mrusso@apigee.com>
Committed: Thu Mar 31 22:05:27 2016 -0700

----------------------------------------------------------------------
 .../main/resources/usergrid-default.properties  | 10 +++
 .../rest/applications/ApplicationCreateIT.java  |  3 +-
 .../collection/users/PermissionsResourceIT.java |  2 +
 .../rest/test/resource/AbstractRestIT.java      |  4 +
 .../usergrid/security/shiro/ShiroCache.java     | 41 ++++++++--
 .../security/shiro/ShiroCacheManager.java       | 15 +++-
 .../security/shiro/utils/LocalShiroCache.java   | 80 ++++++++++++++++++++
 .../resources/usergrid-services-context.xml     | 11 ++-
 .../resources/usergrid-custom-test.properties   |  1 +
 9 files changed, 151 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/9b56fb2c/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index de5e94f..e2ab816 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -591,6 +591,16 @@ usergrid.version.build=${version}
 #
 usergrid.auth.cache.time-to-live=3600
 
+# This setting determines the inmemory cache TTL.  Set this low so updates in other regions
will be picked up on all
+# servers after this in memory cache expires
+#
+usergrid.auth.cache.inmemory.time-to-live=30
+
+# Usergrid will keep an in memory cache so it does not have to get from the distributed cache
in Cassandra as often.
+# This property will allow the cache to be tuned based on JVM memory settings.
+#
+usergrid.auth.cache.inmemory.size=3000
+
 # Default connection types to return for an entity (fewer connection types increases performance)
 # none, in (only connections inbound to the entity), out (only connections outbound from
the entity),
 # all (= in + out)'

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9b56fb2c/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationCreateIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationCreateIT.java
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationCreateIT.java
index cfcc91d..0f9be30 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationCreateIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationCreateIT.java
@@ -18,6 +18,7 @@
 package org.apache.usergrid.rest.applications;
 
 
+import net.jcip.annotations.NotThreadSafe;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.usergrid.rest.test.resource.AbstractRestIT;
 import org.apache.usergrid.rest.test.resource.endpoints.mgmt.ManagementResponse;
@@ -36,7 +37,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
 
-
+@NotThreadSafe
 public class ApplicationCreateIT extends AbstractRestIT {
     private static final Logger logger = LoggerFactory.getLogger(ApplicationCreateIT.class);
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9b56fb2c/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java
b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java
index 973de55..9f6c4e4 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.rest.applications.collection.users;
 import java.util.List;
 import java.util.UUID;
 
+import net.jcip.annotations.NotThreadSafe;
 import org.apache.usergrid.rest.test.resource.AbstractRestIT;
 import org.apache.usergrid.rest.test.resource.model.*;
 import org.apache.usergrid.services.exceptions.ServiceResourceNotFoundException;
@@ -38,6 +39,7 @@ import static org.junit.Assert.*;
 /**
  * Tests permissions of adding and removing users from roles as well as groups.
  */
+@NotThreadSafe
 public class PermissionsResourceIT extends AbstractRestIT {
 
     private static final String ROLE = "permtestrole";

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9b56fb2c/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/AbstractRestIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/AbstractRestIT.java
b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/AbstractRestIT.java
index b25eb92..d2f572f 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/AbstractRestIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/AbstractRestIT.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.rest.test.resource;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.usergrid.cassandra.ClearShiroSubject;
 import org.apache.usergrid.rest.TomcatRuntime;
 import org.apache.usergrid.rest.test.resource.endpoints.ApplicationsResource;
 import org.apache.usergrid.rest.test.resource.endpoints.NamedResource;
@@ -60,6 +61,9 @@ public class AbstractRestIT extends JerseyTest {
     public static TomcatRuntime tomcatRuntime = TomcatRuntime.getInstance();
 
     @Rule
+    public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
+
+    @Rule
     public ClientSetup clientSetup = new ClientSetup( this.getBaseURI().toString() );
 
     protected static final Application descriptor = new Application();

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9b56fb2c/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
b/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
index 4fe569f..88466bf 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
@@ -20,19 +20,18 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import org.apache.shiro.cache.Cache;
 import org.apache.shiro.cache.CacheException;
 import org.apache.shiro.subject.SimplePrincipalCollection;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.persistence.cache.CacheFactory;
 import org.apache.usergrid.persistence.cache.CacheScope;
 import org.apache.usergrid.persistence.cache.ScopedCache;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.apache.usergrid.security.shiro.principals.*;
+import org.apache.usergrid.security.shiro.utils.LocalShiroCache;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Set;
-import java.util.UUID;
 
 
 /**
@@ -45,20 +44,44 @@ public class ShiroCache<K, V> implements Cache<K,V> {
     private final CacheFactory<String, V> cacheFactory;
     private final TypeReference typeRef;
     private final Integer cacheTtl;
+    private final LocalShiroCache localShiroCache;
 
-    public ShiroCache( TypeReference typeRef, CacheFactory<String, V> cacheFactory,
Integer cacheTtl ) {
+    public ShiroCache(TypeReference typeRef, CacheFactory<String, V> cacheFactory,
Integer cacheTtl, LocalShiroCache<K,V> localShiroCache) {
         this.typeRef = typeRef;
         this.cacheFactory = cacheFactory;
         this.cacheTtl = cacheTtl;
+        this.localShiroCache = localShiroCache;
     }
 
     @Override
     public V get(K key) throws CacheException {
         if ( cacheTtl == 0 ) return null;
 
+        V value;
+
+        //check cache first
+        value = (V) localShiroCache.get(getKeyString(key));
+        if( value !=null ){
+            if(logger.isTraceEnabled()) {
+                logger.trace("Shiro value served from local cache: {}", value);
+            }
+            return value;
+
+        }
+
         ScopedCache<String, V> scopedCache = getCacheScope(key);
         if ( scopedCache != null ) {
-            V value = scopedCache.get(getKeyString(key), typeRef);
+
+            value = scopedCache.get(getKeyString(key), typeRef);
+
+            if(value != null) {
+
+                if(logger.isTraceEnabled()) {
+                    logger.trace("Shiro value service from cassandra cache: {}", value);
+                }
+
+                localShiroCache.put(getKeyString(key), value);
+            }
 
             if ( logger.isTraceEnabled() ) {
                 if (value instanceof UsergridAuthorizationInfo) {
@@ -73,9 +96,9 @@ public class ShiroCache<K, V> implements Cache<K,V> {
                 }
             }
 
-            return value;
         }
-        return null;
+
+        return value;
     }
 
     @Override
@@ -84,7 +107,9 @@ public class ShiroCache<K, V> implements Cache<K,V> {
 
         ScopedCache<String, V> scopedCache = getCacheScope(key);
         if ( scopedCache != null ) {
+
             V ret = scopedCache.put(getKeyString(key), value, cacheTtl);
+            localShiroCache.invalidate(getKeyString(key));
 
             if ( logger.isTraceEnabled() ) {
                 if (value instanceof UsergridAuthorizationInfo) {
@@ -96,7 +121,6 @@ public class ShiroCache<K, V> implements Cache<K,V> {
                     logger.trace("Put to AUTHC cache {} for app {}", getKeyString(key), info.toString());
                 }
             }
-
             return ret;
         }
         return null;
@@ -109,12 +133,15 @@ public class ShiroCache<K, V> implements Cache<K,V> {
         ScopedCache<String, V> scopedCache = getCacheScope(key);
         if ( scopedCache != null ) {
             scopedCache.remove( getKeyString(key) );
+
         }
+        localShiroCache.invalidate(getKeyString(key));
         return null;
     }
 
     @Override
     public void clear() throws CacheException {
+        localShiroCache.invalidateAll();
         // no-op: Usergrid logic will invalidate cache as necessary
     }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9b56fb2c/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCacheManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCacheManager.java
b/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCacheManager.java
index a3c3062..91befb0 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCacheManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCacheManager.java
@@ -24,6 +24,7 @@ import org.apache.shiro.cache.Cache;
 import org.apache.shiro.cache.CacheException;
 import org.apache.shiro.cache.CacheManager;
 import org.apache.usergrid.persistence.cache.CacheFactory;
+import org.apache.usergrid.security.shiro.utils.LocalShiroCache;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -45,14 +46,21 @@ public class ShiroCacheManager implements CacheManager {
 
     private Map<String, ShiroCache> caches = new HashMap<>();
 
+    @Autowired
     private Properties properties;
 
     private Integer cacheTtl = null; // specified in seconds
 
     private static final String CACHE_TTL_PROPERTY_NAME = "usergrid.auth.cache.time-to-live";
 
+    private LocalShiroCache localShiroCache;
 
-    public ShiroCacheManager() {}
+    public ShiroCacheManager(Injector injector, Properties properties) {
+
+        this.injector = injector;
+        this.properties = properties;
+        this.localShiroCache = injector.getInstance(LocalShiroCache.class);
+    }
 
 
     @Override
@@ -68,7 +76,7 @@ public class ShiroCacheManager implements CacheManager {
                 shiroCache = new ShiroCache(
                     new TypeReference<UsergridAuthorizationInfo>() {},
                     (CacheFactory)injector.getInstance( Key.get(typeLit) ),
-                    getCacheTtl());
+                    getCacheTtl(), localShiroCache);
 
             } else if ("realm.authenticationCache".equals(name)) {
 
@@ -77,7 +85,7 @@ public class ShiroCacheManager implements CacheManager {
                 shiroCache = new ShiroCache(
                     new TypeReference<UsergridAuthenticationInfo>() {},
                     (CacheFactory)injector.getInstance( Key.get(typeLit) ),
-                    getCacheTtl());
+                    getCacheTtl(), localShiroCache);
 
             } else {
                 logger.error("Unknown Shiro Cache name: {}", name);
@@ -106,7 +114,6 @@ public class ShiroCacheManager implements CacheManager {
         return properties;
     }
 
-    @Autowired
     public void setProperties(Properties properties) {
         this.properties = properties;
     }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9b56fb2c/stack/services/src/main/java/org/apache/usergrid/security/shiro/utils/LocalShiroCache.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/utils/LocalShiroCache.java
b/stack/services/src/main/java/org/apache/usergrid/security/shiro/utils/LocalShiroCache.java
new file mode 100644
index 0000000..019a457
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/utils/LocalShiroCache.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.security.shiro.utils;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.inject.Singleton;
+import org.apache.usergrid.security.shiro.ShiroCacheManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+
+@Singleton
+public class LocalShiroCache<K, V> {
+
+    private final Cache<K,V> cache;
+
+    private static final String CACHE_TTL_PROP = "usergrid.auth.cache.inmemory.time-to-live";
+    private static final String CACHE_MAX_SIZE_PROP = "usergrid.auth.cache.inmemory.size";
+
+
+    public LocalShiroCache(){
+
+        // set a smaller ttl
+        long ttl = 1;
+        int configuredMaxSize;
+
+        try{
+            ttl = Integer.parseInt(System.getProperty(CACHE_TTL_PROP));
+        } catch (NumberFormatException e){
+           // already defaulted to 1 above
+        }
+
+        try{
+            configuredMaxSize = Integer.parseInt(System.getProperty(CACHE_MAX_SIZE_PROP));
+        } catch (NumberFormatException e){
+            configuredMaxSize = 1000;
+        }
+
+        this.cache = CacheBuilder.newBuilder()
+            .maximumSize(Math.max(1000,configuredMaxSize))
+            .expireAfterWrite(ttl, TimeUnit.SECONDS).build();
+
+    }
+
+    public void put(K key, V value){
+
+        cache.put(key, value);
+    }
+
+
+    public V get(K key){
+        return cache.getIfPresent(key);
+    }
+
+    public void invalidate(K key){
+        cache.invalidate(key);
+    }
+
+    public void invalidateAll(){
+        cache.invalidateAll();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9b56fb2c/stack/services/src/main/resources/usergrid-services-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/main/resources/usergrid-services-context.xml b/stack/services/src/main/resources/usergrid-services-context.xml
index c41080d..c064f60 100644
--- a/stack/services/src/main/resources/usergrid-services-context.xml
+++ b/stack/services/src/main/resources/usergrid-services-context.xml
@@ -34,8 +34,13 @@
     <!--
     <bean id="cacheManager" class="org.apache.shiro.cache.MemoryConstrainedCacheManager"/>
     -->
+    <bean id="injector"
+          class="org.apache.usergrid.corepersistence.GuiceFactory">
+    </bean>
+
     <bean id="cacheManager" class="org.apache.usergrid.security.shiro.ShiroCacheManager">
-        <property name="properties" ref="properties"/>
+        <constructor-arg name="injector" ref="injector"/>
+        <constructor-arg name="properties" ref="properties"/>
     </bean>
 
     <bean id="realm" class="org.apache.usergrid.security.shiro.Realm">
@@ -63,9 +68,7 @@
         <property name="entityManagerFactory" ref="entityManagerFactory"/>
     </bean>
 
-    <bean id="injector"
-          class="org.apache.usergrid.corepersistence.GuiceFactory">
-    </bean>
+
 
     <bean id="managementService" class="org.apache.usergrid.management.cassandra.ManagementServiceImpl">
         <property name="saltProvider" ref="saltProvider"/>

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9b56fb2c/stack/services/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/usergrid-custom-test.properties b/stack/services/src/test/resources/usergrid-custom-test.properties
index 1c3d190..9a800c9 100644
--- a/stack/services/src/test/resources/usergrid-custom-test.properties
+++ b/stack/services/src/test/resources/usergrid-custom-test.properties
@@ -42,3 +42,4 @@ usergrid.cluster_name=usergrid
 
 # specified in seconds
 usergrid.auth.cache.time-to-live=1
+usergrid.auth.cache.inmemory.time-to-live=1


Mime
View raw message