usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [1/2] git commit: Use EntityManager instead of EntityCollectionManager to repersist.
Date Fri, 10 Oct 2014 22:41:05 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/two-dot-o 884b0300d -> 56d703898


Use EntityManager instead of EntityCollectionManager to repersist.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5a3ece54
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5a3ece54
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5a3ece54

Branch: refs/heads/two-dot-o
Commit: 5a3ece54eb81859d9ce40f98416e7fd8239fc6ef
Parents: 693cb2d
Author: Dave Johnson <dmjohnson@apigee.com>
Authored: Fri Oct 10 18:14:54 2014 -0400
Committer: Dave Johnson <dmjohnson@apigee.com>
Committed: Fri Oct 10 18:14:54 2014 -0400

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        |  87 ++-----
 .../corepersistence/CpEntityManagerFactory.java |  22 --
 .../usergrid/corepersistence/CpVisitor.java     |   8 +-
 .../usergrid/corepersistence/CpWalker.java      | 144 +++--------
 .../HybridEntityManagerFactory.java             |  10 -
 .../corepersistence/results/EntityVerifier.java |   3 +-
 .../results/FilteringLoader.java                |   7 +-
 .../usergrid/persistence/EntityManager.java     |   3 -
 .../persistence/EntityManagerFactory.java       |   4 -
 .../cassandra/EntityManagerFactoryImpl.java     |  10 -
 .../cassandra/EntityManagerImpl.java            |   7 +-
 .../PerformanceEntityRepersistTest.java         | 236 -------------------
 .../impl/EntityCollectionManagerImpl.java       |  36 +--
 stack/pom.xml                                   |   4 +-
 .../apache/usergrid/rest/SystemResource.java    |  30 ++-
 .../org/apache/usergrid/tools/RepersistAll.java |  98 --------
 16 files changed, 98 insertions(+), 611 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 548afda..88cee9c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2862,85 +2862,30 @@ public class CpEntityManager implements EntityManager {
 
             @Override
             public void visitCollectionEntry(
-                EntityCollectionManager ecm,
-                String collectionName, 
-                org.apache.usergrid.persistence.model.entity.Entity collectionEntity, 
-                org.apache.usergrid.persistence.model.entity.Entity memberEntity) {
-
-                EntityRef source = new SimpleEntityRef( 
-                        collectionEntity.getId().getType(), collectionEntity.getId().getUuid() );
-                EntityRef target = new SimpleEntityRef( 
-                        memberEntity.getId().getType(), memberEntity.getId().getUuid() );
-                po.onProgress(source, target, "dummy");
-
-                indexEntityIntoCollection( 
-                    collectionEntity, memberEntity, collectionName );
-            }
-
-            @Override
-            public void visitConnectionEntry(
-                EntityCollectionManager ecm,
-                String connectionType, 
-                org.apache.usergrid.persistence.model.entity.Entity sourceEntity, 
-                org.apache.usergrid.persistence.model.entity.Entity targetEntity) {
-
-                EntityRef source = new SimpleEntityRef( 
-                        sourceEntity.getId().getType(), sourceEntity.getId().getUuid() );
-                EntityRef target = new SimpleEntityRef( 
-                        targetEntity.getId().getType(), targetEntity.getId().getUuid() );
-                po.onProgress(source, target, "dummy");
-
-                indexEntityIntoConnection( 
-                    sourceEntity, targetEntity, connectionType );
-            }
-        });
-    }
-
-
-    /** 
-     * Completely repersist the application associated with this EntityManager.
-     */
-    public void repersistApplication( 
-            final UUID appId, final EntityManagerFactory.ProgressObserver po ) throws Exception {
+                    EntityManager em, String collName, Entity source, Entity target) {
 
-        CpWalker walker = new CpWalker();
+                try {
+                    em.update( target);
+                    po.onProgress(source, target, collName);
 
-        walker.walkCollections( this, application, new CpVisitor() {
+                } catch (Exception ex) {
+                    logger.error("Error repersisting entity", ex);
+                }
+            }
 
             @Override
-            public void visitCollectionEntry(
-                EntityCollectionManager ecm,
-                String collectionName, 
-                org.apache.usergrid.persistence.model.entity.Entity collectionEntity, 
-                org.apache.usergrid.persistence.model.entity.Entity memberEntity) {
-
-                EntityRef source = new SimpleEntityRef( 
-                        collectionEntity.getId().getType(), collectionEntity.getId().getUuid() );
-                EntityRef target = new SimpleEntityRef( 
-                        memberEntity.getId().getType(), memberEntity.getId().getUuid() );
-                po.onProgress(source, target, "dummy");
+            public void visitConnectionEntry(
+                    EntityManager em, String connType, Entity source, Entity target) {
 
-                ecm.write( memberEntity).toBlocking().last();
+                try {
+                    em.update( target);
+                    po.onProgress(source, target, connType);
 
-                indexEntityIntoCollection( 
-                    collectionEntity, memberEntity, collectionName );
+                } catch (Exception ex) {
+                    logger.error("Error repersisting entity", ex);
+                }
             }
 
-            @Override
-            public void visitConnectionEntry(
-                EntityCollectionManager ecm,
-                String connectionType, 
-                org.apache.usergrid.persistence.model.entity.Entity sourceEntity, 
-                org.apache.usergrid.persistence.model.entity.Entity targetEntity) {
-
-                EntityRef source = new SimpleEntityRef( 
-                        sourceEntity.getId().getType(), sourceEntity.getId().getUuid() );
-                EntityRef target = new SimpleEntityRef( 
-                        targetEntity.getId().getType(), targetEntity.getId().getUuid() );
-                po.onProgress(source, target, "dummy");
-
-                ecm.write( targetEntity).toBlocking().last();
-            }
         });
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index cf19050..1048505 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -699,26 +699,4 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
     public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po ) {
         throw new UnsupportedOperationException( "Not supported yet." );
     }
-
-    @Override
-    public void repersistApplication( UUID appId, ProgressObserver po ) throws Exception {
-        EntityManager em = getEntityManager( appId );
-        em.repersistApplication( appId, po );
-    }
-
-    @Override
-    public void repersistAll( ProgressObserver po ) throws Exception {
-
-        logger.info("\n\nRepersisting all Entities\n");
-
-        Map<String, UUID> appMap = getApplications();
-
-        logger.info("About to repersist entities for {} applications", appMap.keySet().size());
-
-        for ( UUID appUuid : appMap.values() ) {
-            repersistApplication( appUuid, po );
-        }
-
-    }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpVisitor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpVisitor.java
index 02d13c1..5d32235 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpVisitor.java
@@ -15,8 +15,8 @@
  */
 package org.apache.usergrid.corepersistence;
 
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
 
 
 /**
@@ -25,8 +25,8 @@ import org.apache.usergrid.persistence.model.entity.Entity;
 public interface CpVisitor {
 
     public void visitCollectionEntry( 
-        EntityCollectionManager ecm, String collName, Entity collEntity, Entity memberEntity );
+        EntityManager em, String collName, Entity sourceEntity, Entity targetEntity );
 
     public void visitConnectionEntry( 
-        EntityCollectionManager ecm, String connType, Entity sourceEntity, Entity targetEntity );
+        EntityManager em, String connType, Entity sourceEntity, Entity targetEntity );
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
index e780b77..d3f668d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
@@ -16,14 +16,12 @@
 package org.apache.usergrid.corepersistence;
 
 import java.util.Stack;
-import java.util.logging.Level;
 
 import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.Entity;
 import org.apache.usergrid.persistence.EntityRef;
 import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.SimpleEntityRef;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.graph.Edge;
 import org.apache.usergrid.persistence.graph.GraphManager;
@@ -101,127 +99,53 @@ public class CpWalker {
                     @Override
                     public void call( Edge edge ) {
 
+                        EntityRef sourceEntityRef = new SimpleEntityRef( 
+                            edge.getSourceNode().getType(), edge.getSourceNode().getUuid());
+                        Entity sourceEntity;
+                        try {
+                            sourceEntity = em.get( sourceEntityRef );
+                        } catch (Exception ex) {
+                            logger.error( "Error getting sourceEntity {}:{}, continuing", 
+                                    sourceEntityRef.getType(), sourceEntityRef.getUuid());
+                            return;
+                        }
+
+                        EntityRef targetEntityRef = new SimpleEntityRef( 
+                            edge.getTargetNode().getType(), edge.getTargetNode().getUuid());
+                        Entity targetEntity;
+                        try {
+                            targetEntity = em.get( targetEntityRef );
+                        } catch (Exception ex) {
+                            logger.error( "Error getting sourceEntity {}:{}, continuing", 
+                                    sourceEntityRef.getType(), sourceEntityRef.getUuid());
+                            return;
+                        }
+                            
                         if ( CpNamingUtils.isCollectionEdgeType( edge.getType() )) {
 
                             String collName = CpNamingUtils.getCollectionName( edgeType );
-                            String memberType = edge.getTargetNode().getType();
-
-                            CollectionScope collScope = new CollectionScopeImpl(
-                                applicationScope.getApplication(),
-                                applicationScope.getApplication(),
-                                CpNamingUtils.getCollectionScopeNameFromEntityType( start.getType()));
-                            EntityCollectionManager collMgr = 
-                                em.getManagerCache().getEntityCollectionManager(collScope);
-
-                            org.apache.usergrid.persistence.model.entity.Entity collEntity = 
-                                collMgr.load( edge.getSourceNode() ).toBlockingObservable().last();
-
-                            if (collEntity == null) {
-                                logger.warn("(Empty collection?) Failed to load collection entity "
-                                        + "{}:{} from scope\n   app {}\n   owner {}\n   name {}",
-                                        new Object[]{
-                                            edge.getSourceNode().getType(), 
-                                            edge.getSourceNode().getUuid(),
-                                            collScope.getApplication(),
-                                            collScope.getOwner(),
-                                            collScope.getName()
-                                        });
-                                return;
-                            }
-
-                            CollectionScope memberScope = new CollectionScopeImpl(
-                                applicationScope.getApplication(),
-                                applicationScope.getApplication(),
-                                CpNamingUtils.getCollectionScopeNameFromEntityType( memberType ));
-                            EntityCollectionManager memberMgr = 
-                                em.getManagerCache().getEntityCollectionManager(memberScope);
-
-                            org.apache.usergrid.persistence.model.entity.Entity memberEntity = 
-                                memberMgr.load( edge.getTargetNode()).toBlockingObservable().last();
-
-                            if (memberEntity == null) {
-                                logger.warn("(Empty collection?) Failed to load member entity "
-                                        + "{}:{} from scope\n   app {}\n   owner {}\n   name {}",
-                                        new Object[]{
-                                            edge.getTargetNode().getType(), 
-                                            edge.getTargetNode().getUuid(),
-                                            memberScope.getApplication(),
-                                            memberScope.getOwner(),
-                                            memberScope.getName()
-                                        });
-                                return;
-                            }
 
                             visitor.visitCollectionEntry( 
-                                    memberMgr, collName, collEntity, memberEntity );
+                                    em, collName, sourceEntity, targetEntity );
 
                             // recursion
-                            if ( !stack.contains( memberEntity.getId() )) {
-                                stack.push( memberEntity.getId() );
-                                doWalkCollections( em, memberEntity.getId(), visitor, stack );
+                            if ( !stack.contains( targetEntity.getUuid() )) {
+                                stack.push( targetEntity.getUuid() );
+                                doWalkCollections( em, edge.getSourceNode(), visitor, stack );
                                 stack.pop(); 
                             }
 
+                        } else {
 
-                        } else if ( CpNamingUtils.isConnectionEdgeType( edge.getType() )) {
-
-                            String connType = CpNamingUtils.getConnectionType( edgeType );
-                            String targetEntityType = edge.getTargetNode().getType();
-                            String sourceEntityType = start.getType();
-
-                            CollectionScope sourceScope = new CollectionScopeImpl(
-                                applicationScope.getApplication(),
-                                applicationScope.getApplication(),
-                                CpNamingUtils.getCollectionScopeNameFromEntityType(sourceEntityType));
-                            EntityCollectionManager sourceEcm = 
-                                em.getManagerCache().getEntityCollectionManager(sourceScope);
-
-                            org.apache.usergrid.persistence.model.entity.Entity sourceEntity = 
-                                sourceEcm.load( edge.getSourceNode() ).toBlockingObservable().last();
-
-                            if (sourceEntity == null) {
-                                logger.warn("(Empty connection?) Failed to load source entity "
-                                        + "{}:{} from scope\n   app {}\n   owner {}\n   name {}", 
-                                        new Object[]{
-                                            edge.getSourceNode().getType(), 
-                                            edge.getSourceNode().getUuid(),
-                                            sourceScope.getApplication(),
-                                            sourceScope.getOwner(),
-                                            sourceScope.getName()
-                                        });
-                                return;
-                            }
-
-                            CollectionScope targetScope = new CollectionScopeImpl(
-                                applicationScope.getApplication(),
-                                applicationScope.getApplication(),
-                                CpNamingUtils.getCollectionScopeNameFromEntityType(targetEntityType));
-                            EntityCollectionManager targetEcm = 
-                                em.getManagerCache().getEntityCollectionManager(targetScope);
-
-                            org.apache.usergrid.persistence.model.entity.Entity targetEntity = 
-                                targetEcm.load( edge.getTargetNode() ).toBlockingObservable().last();
-
-                            if (targetEntity == null) {
-                                logger.warn("(Empty connection?) Failed to load target entity "
-                                        + "{}:{} from scope\n   app {}\n   owner {}\n   name {}",
-                                        new Object[]{
-                                            edge.getTargetNode().getType(), 
-                                            edge.getTargetNode().getUuid(),
-                                            targetScope.getApplication(),
-                                            targetScope.getOwner(),
-                                            targetScope.getName()
-                                        });
-                                return;
-                            }
+                            String collName = CpNamingUtils.getConnectionType(edgeType);
 
                             visitor.visitConnectionEntry( 
-                                    targetEcm, connType, sourceEntity, targetEntity );
+                                    em, collName, sourceEntity, targetEntity );
 
                             // recursion
-                            if ( !stack.contains( targetEntity.getId() )) {
-                                stack.push( targetEntity.getId() );
-                                doWalkCollections( em, targetEntity.getId(), visitor, stack );
+                            if ( !stack.contains( targetEntity.getUuid() )) {
+                                stack.push( targetEntity.getUuid() );
+                                doWalkCollections( em, edge.getTargetNode(), visitor, stack );
                                 stack.pop(); 
                             }
                         }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index adc0945..54a5dee 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -163,14 +163,4 @@ public class HybridEntityManagerFactory implements EntityManagerFactory, Applica
     public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po) {
         factory.rebuildCollectionIndex(appId, collection, po);
     }
-
-    @Override
-    public void repersistAll( ProgressObserver po ) throws Exception {
-        factory.repersistAll( po );
-    }
-
-    @Override
-    public void repersistApplication(UUID appId, ProgressObserver po) throws Exception {
-        factory.repersistApplication( appId, po );
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
index 31018b6..a35263c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
@@ -65,6 +65,7 @@ public class EntityVerifier implements ResultsVerifier {
     @Override
     public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
         ids = ecm.load( idsToLoad ).toBlocking().last();
+        logger.debug("loadResults() asked for {} ids and got {}", idsToLoad.size(), ids.size());
     }
 
 
@@ -76,7 +77,7 @@ public class EntityVerifier implements ResultsVerifier {
 
         //version wasn't found deindex
         if ( savedEntity == null ) {
-            logger.warn( "Version for Entity {}:{} not found", entityId.getUuid(), entityId.getUuid() );
+            logger.warn( "Version for Entity {}:{} not found", entityId.getType(), entityId.getUuid() );
             return false;
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 4973783..757fec2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -189,12 +189,9 @@ public class FilteringLoader implements ResultsLoader {
 
             //now using the scope, load the collection
 
-
             // Get the collection scope and batch load all the versions
-            final CollectionScope collScope =
-                    new CollectionScopeImpl( applicationScope.getApplication(), applicationScope.getApplication(),
-                            scopeName );
-
+            final CollectionScope collScope = new CollectionScopeImpl( 
+                    applicationScope.getApplication(), ownerId, scopeName );
 
             final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collScope );
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 69a3b3f..3684d7e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -694,8 +694,5 @@ public interface EntityManager {
     /** For testing purposes */
     public void flushManagerCaches();
 
-    public void repersistApplication( 
-            final UUID appId, final EntityManagerFactory.ProgressObserver po ) throws Exception;
-
     public void reindex( final EntityManagerFactory.ProgressObserver po ) throws Exception;
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 714db51..06c3114 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -130,10 +130,6 @@ public interface EntityManagerFactory {
 
     public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver object);
 
-    public void repersistAll( ProgressObserver po ) throws Exception;
-
-    public void repersistApplication( UUID appId, ProgressObserver po ) throws Exception;
-
     public interface ProgressObserver {
         public void onProgress( EntityRef source, EntityRef target, String edgeType );
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 26e55aa..399bccd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -438,14 +438,4 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
     public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po) {
         throw new UnsupportedOperationException("Not supported."); 
     }
-
-    @Override
-    public void repersistAll( ProgressObserver po ) {
-        throw new UnsupportedOperationException("Not supported."); 
-    }
-
-    @Override
-    public void repersistApplication(UUID appId, ProgressObserver po) throws Exception {
-        throw new UnsupportedOperationException("Not supported yet."); 
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index d44dac3..7437c7a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -2923,12 +2923,7 @@ public class EntityManagerImpl implements EntityManager {
     public void flushManagerCaches() {
         // no-op
     }
-
-    @Override
-    public void repersistApplication(UUID appId, EntityManagerFactory.ProgressObserver po) throws Exception {
-        throw new UnsupportedOperationException("Not supported."); 
-    }
-
+    
     @Override
     public void reindex(EntityManagerFactory.ProgressObserver po) throws Exception {
         throw new UnsupportedOperationException("Not supported."); 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRepersistTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRepersistTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRepersistTest.java
deleted file mode 100644
index 5b4cccc..0000000
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRepersistTest.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.persistence;
-
-
-import java.util.concurrent.TimeUnit;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.AbstractCoreIT;
-import org.apache.usergrid.Application;
-import org.apache.usergrid.CoreApplication;
-
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.Slf4jReporter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.usergrid.cassandra.Concurrent;
-import org.apache.usergrid.persistence.index.query.Query;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-
-//@RunWith(JukitoRunner.class)
-//@UseModules({ GuiceModule.class })
-@Concurrent()
-public class PerformanceEntityRepersistTest extends AbstractCoreIT {
-    private static final Logger logger = 
-            LoggerFactory.getLogger(PerformanceEntityRepersistTest.class );
-
-    private static final MetricRegistry registry = new MetricRegistry();
-    private Slf4jReporter reporter;
-
-    private static final long RUNTIME = TimeUnit.MINUTES.toMillis( 1 );
-
-    private static final long writeDelayMs = 100;
-    //private static final long readDelayMs = 7;
-
-    @Rule
-    public Application app = new CoreApplication( setup );
-
-
-    @Before
-    public void startReporting() {
-
-        logger.debug("Starting metrics reporting");
-        reporter = Slf4jReporter.forRegistry( registry ).outputTo( logger )
-                .convertRatesTo( TimeUnit.SECONDS )
-                .convertDurationsTo( TimeUnit.MILLISECONDS ).build();
-
-        reporter.start( 10, TimeUnit.SECONDS );
-    }
-
-
-    @After
-    public void printReport() {
-
-        logger.debug("Printing metrics report");
-        reporter.report();
-        reporter.stop();
-    }
-
-
-    @Test
-    public void repersistAll() throws Exception {
-
-        logger.info("Started repersistAll()");
-
-        final EntityManager em = app.getEntityManager();
-
-        // ----------------- create a bunch of entities
-
-        final long stopTime = System.currentTimeMillis() + RUNTIME;
-
-        Map<String, Object> entityMap = new HashMap<String, Object>() {{
-            put("key1", 1000 );
-            put("key2", 2000 );
-            put("key3", "Some value");
-        }};
-        Map<String, Object> cat1map = new HashMap<String, Object>() {{
-            put("name", "enzo");
-            put("color", "orange");
-        }};
-        Map<String, Object> cat2map = new HashMap<String, Object>() {{
-            put("name", "marquee");
-            put("color", "grey");
-        }};
-        Map<String, Object> cat3map = new HashMap<String, Object>() {{
-            put("name", "bertha");
-            put("color", "tabby");
-        }};
-
-        Entity cat1 = em.create("cat", cat1map );
-        Entity cat2 = em.create("cat", cat2map );
-        Entity cat3 = em.create("cat", cat3map );
-
-        List<EntityRef> entityRefs = new ArrayList<EntityRef>();
-        int entityCount = 0;
-        while ( System.currentTimeMillis() < stopTime ) {
-
-            final Entity entity;
-
-            try {
-                entityMap.put("key", entityCount );
-                entity = em.create("testType", entityMap );
-
-//                em.refreshIndex();
-
-                em.createConnection(entity, "herds", cat1);
-                em.createConnection(entity, "herds", cat2);
-                em.createConnection(entity, "herds", cat3);
-
-            } catch (Exception ex) {
-                throw new RuntimeException("Error creating entity", ex);
-            }
-
-            entityRefs.add(new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
-            if ( entityCount % 100 == 0 ) {
-                logger.info("Created {} entities", entityCount );
-            }
-
-            entityCount++;
-            try { Thread.sleep( writeDelayMs ); } catch (InterruptedException ignored ) {}
-        }
-
-        logger.info("Created {} entities", entityCount);
-        em.refreshIndex();
-
-        // ----------------- test that we can read them, should work fine 
-
-        logger.debug("Read the data");
-        readData("testTypes", entityCount );
-
-        // ----------------- repersist all
-
-        logger.debug("Preparing to repersist all");;
-
-        final String meterName = this.getClass().getSimpleName() + ".repersist";
-        final Meter meter = registry.meter( meterName );
-        
-        EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
-            int counter = 0;
-            @Override
-            public void onProgress( EntityRef s, EntityRef t, String etype ) {
-                meter.mark();
-                logger.debug("Repersisting from {}:{} to {}:{} ", new Object[] {
-                    s.getType(), s.getUuid(), t.getType(), t.getUuid(), etype });
-                if ( !logger.isDebugEnabled() && counter % 100 == 0 ) {
-                    logger.info("Repersisted {} entities", counter );
-                }
-                counter++;
-            }
-        };
-
-        try {
-            setup.getEmf().repersistAll(po );
-
-            registry.remove( meterName );
-            logger.info("Repersist complete");
-
-        } catch (Exception ex) {
-            logger.error("Error repersisting", ex);
-            fail();
-        }
-        em.refreshIndex();
-
-        // ----------------- test that we can read them
-        
-        readData( "testTypes", entityCount );
-    }
-
-
-
-    private int readData( String collectionName, int expected ) throws Exception {
-
-        EntityManager em = app.getEntityManager();
-
-        Query q = Query.fromQL("select * where key1=1000");
-        q.setLimit(40);
-        Results results = em.searchCollection( em.getApplicationRef(), collectionName, q );
-
-        int count = 0;
-        while ( true ) {
-
-            for ( Entity e : results.getEntities() ) {
-
-                assertEquals( 2000, e.getProperty("key2"));
-
-                Results catResults = em.searchConnectedEntities(e, Query.fromQL("select *"));
-                assertEquals( 3, catResults.size() );
-
-                if ( count % 100 == 0 ) {
-                    logger.info( "read {} entities", count);
-                }
-                count++;
-            }
-
-            if ( results.hasCursor() ) {
-                logger.info( "Counted {} : query again with cursor", count);
-                q.setCursor( results.getCursor() );
-                results = em.searchCollection( em.getApplicationRef(), collectionName, q );
-
-            } else {
-                break;
-            }
-        }
-
-        if ( expected != -1 && expected != count ) {
-            throw new RuntimeException("Did not get expected " 
-                    + expected + " entities, instead got " + count );
-        }
-        return count;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index c57f666..9f3c1ea 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -326,23 +326,23 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
 
     @Override
 
-    public Observable<VersionSet> getLatestVersion( final Collection<Id> entityIds ) {
-
-        return Observable.create( new Observable.OnSubscribe<VersionSet>() {
-
-                  @Override
-                  public void call( final Subscriber<? super VersionSet> subscriber ) {
-                      try {
-                          final  VersionSet logEntries = mvccLogEntrySerializationStrategy.load( collectionScope, entityIds,
-                                          UUIDGenerator.newTimeUUID() );
-
-                          subscriber.onNext( logEntries );
-                          subscriber.onCompleted();
-                      }
-                      catch ( Exception e ) {
-                          subscriber.onError( e );
-                      }
-                  }
-              } ); 
+    public Observable<VersionSet> getLatestVersion(final Collection<Id> entityIds) {
+
+        return Observable.create(new Observable.OnSubscribe<VersionSet>() {
+
+            @Override
+            public void call(final Subscriber<? super VersionSet> subscriber) {
+                try {
+                    final VersionSet logEntries = mvccLogEntrySerializationStrategy
+                        .load(collectionScope, entityIds, UUIDGenerator.newTimeUUID());
+
+                    subscriber.onNext(logEntries);
+                    subscriber.onCompleted();
+
+                } catch (Exception e) {
+                    subscriber.onError(e);
+                }
+            }
+        });
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index b1bfe3c..9231a2f 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -199,14 +199,16 @@
     <module>core</module>
     <module>services</module>
     <module>rest</module>
+    <!--
     <module>tools</module>
     <module>websocket</module>
+    -->
     <!--
     Re-enable when we have a fix for the AppleJavaExtensions jar issue
     https://issues.apache.org/jira/browse/USERGRID-224
     <module>launcher</module>
     -->
-    <module>mongo-emulator</module>
+    <!-- <module>mongo-emulator</module> -->
     <!--  
     Re-enable when query-validator updated to work with Core Persistence.
     https://issues.apache.org/jira/browse/USERGRID-221

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
index b037b96..ceec656 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
@@ -43,6 +43,7 @@ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
 import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
 
 import com.sun.jersey.api.json.JSONWithPadding;
+import org.apache.usergrid.persistence.EntityManagerFactory.ProgressObserver;
 
 
 @Path( "/system" )
@@ -66,7 +67,7 @@ public class SystemResource extends AbstractContextResource {
     @GET
     @Path( "database/setup" )
     public JSONWithPadding getSetup( @Context UriInfo ui,
-                                     @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+                             @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
             throws Exception {
 
         ApiResponse response = createApiResponse();
@@ -90,7 +91,7 @@ public class SystemResource extends AbstractContextResource {
     @GET
     @Path( "superuser/setup" )
     public JSONWithPadding getSetupSuperuser( @Context UriInfo ui,
-                                              @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+                             @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
             throws Exception {
 
         ApiResponse response = createApiResponse();
@@ -115,14 +116,14 @@ public class SystemResource extends AbstractContextResource {
     @PUT
     @Path( "index/rebuild" )
     public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
-                                           @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+                             @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
             throws Exception {
 
         ApiResponse response = createApiResponse();
         response.setAction( "rebuild indexes" );
 
 
-        final EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+        final ProgressObserver po = new ProgressObserver() {
             @Override
             public void onProgress( EntityRef s, EntityRef t, String etype ) {
                 logger.info( "Indexing from {}:{} to {}:{} edgeType {}", new Object[] {
@@ -164,8 +165,11 @@ public class SystemResource extends AbstractContextResource {
     @RequireSystemAccess
     @PUT
     @Path( "index/rebuild/" + RootResource.APPLICATION_ID_PATH )
-    public JSONWithPadding rebuildIndexes( @Context UriInfo ui, @PathParam( "applicationId" ) String applicationIdStr,
-                                           @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+    public JSONWithPadding rebuildIndexes( 
+                @Context UriInfo ui, 
+                @PathParam( "applicationId" ) String applicationIdStr,
+                @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+
             throws Exception {
 
         final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
@@ -173,7 +177,7 @@ public class SystemResource extends AbstractContextResource {
         response.setAction( "rebuild indexes" );
 
 
-        final EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+        final ProgressObserver po = new ProgressObserver() {
             @Override
             public void onProgress( EntityRef s, EntityRef t, String etype ) {
                 logger.info( "Indexing from {}:{} to {}:{} edgeType {}", new Object[] {
@@ -213,10 +217,11 @@ public class SystemResource extends AbstractContextResource {
     @RequireSystemAccess
     @PUT
     @Path( "index/rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
-    public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
-                                           @PathParam( "applicationId" ) final String applicationIdStr,
-                                           @PathParam( "collectionName" ) final String collectionName,
-                                           @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+    public JSONWithPadding rebuildIndexes( 
+                @Context UriInfo ui,
+                @PathParam( "applicationId" ) final String applicationIdStr,
+                @PathParam( "collectionName" ) final String collectionName,
+                @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
             throws Exception {
 
         final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
@@ -231,7 +236,8 @@ public class SystemResource extends AbstractContextResource {
             }
         };
 
-        rebuild.setName( String.format( "Index rebuild for app %s and collection %s", appId, collectionName ) );
+        rebuild.setName( String.format( 
+                "Index rebuild for app %s and collection %s", appId, collectionName ) );
         rebuild.setDaemon( true );
         rebuild.start();
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a3ece54/stack/tools/src/main/java/org/apache/usergrid/tools/RepersistAll.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/RepersistAll.java b/stack/tools/src/main/java/org/apache/usergrid/tools/RepersistAll.java
deleted file mode 100644
index 4c63f61..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/RepersistAll.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.EntityRef;
-
-
-/**
- * Index rebuild utility for Usergrid. Can be used to rebuild the index for a specific 
- * application, a specific application's collection or for an entire Usergrid system.
- */
-public class RepersistAll extends ToolBase {
-
-    private static final int PAGE_SIZE = 100;
-
-
-    private static final Logger logger = LoggerFactory.getLogger(RepersistAll.class );
-
-
-    @Override
-    @SuppressWarnings("static-access")
-    public Options createOptions() {
-
-        Option hostOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
-                .withDescription( "Cassandra host" ).create( "host" );
-
-        Option esHostsOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
-                .withDescription( "ElasticSearch host" ).create( "eshost" );
-
-        Option esClusterOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
-                .withDescription( "ElasticSearch cluster name" ).create( "escluster" );
-
-        Options options = new Options();
-        options.addOption( hostOpt );
-        options.addOption( esHostsOpt );
-        options.addOption( esClusterOpt );
-
-        return options;
-    }
-
-
-    /*
-     * (non-Javadoc)
-     * 
-     * @see
-     * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
-     */
-    @Override
-    public void runTool( CommandLine line ) throws Exception {
-        startSpring();
-
-        logger.info( "Starting index rebuild" );
-
-        EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
-            @Override
-            public void onProgress(EntityRef s, EntityRef t, String etype) {
-                logger.info("Repersisting from {}:{} to {}:{}", new Object[] {
-                    s.getType(), s.getUuid(), t.getType(), t.getUuid(), etype });
-            }
-        };
-
-        emf.rebuildAllIndexes( po );
-
-        logger.info( "Finished index rebuild" );
-    }
-}


Mime
View raw message