usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [2/2] git commit: Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Date Fri, 10 Oct 2014 00:47:34 GMT
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
two-dot-o

Conflicts:
	stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
	stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/140e5f79
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/140e5f79
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/140e5f79

Branch: refs/heads/two-dot-o
Commit: 140e5f79316231f51740d24126bd6912de67e760
Parents: cb7cfad 62da6cf
Author: Dave Johnson <dmjohnson@apigee.com>
Authored: Thu Oct 9 20:47:20 2014 -0400
Committer: Dave Johnson <dmjohnson@apigee.com>
Committed: Thu Oct 9 20:47:20 2014 -0400

----------------------------------------------------------------------
 .../corepersistence/CpEntityDeleteListener.java |   2 +-
 .../corepersistence/CpEntityManager.java        |  93 ++++-
 .../corepersistence/CpEntityManagerFactory.java | 194 +++++++---
 .../corepersistence/CpEntityMapUtils.java       | 323 ----------------
 .../corepersistence/CpManagerCache.java         |   2 +-
 .../usergrid/corepersistence/CpNamingUtils.java | 106 ------
 .../corepersistence/CpRelationManager.java      | 140 +------
 .../corepersistence/results/EntityVerifier.java | 142 +++++++
 .../results/FilteringLoader.java                | 245 ++++++++++++
 .../corepersistence/results/IdsVerifier.java    |  65 ++++
 .../corepersistence/results/RefsVerifier.java   |  62 +++
 .../corepersistence/results/ResultsLoader.java  |  58 +++
 .../results/ResultsLoaderFactory.java           |  60 +++
 .../results/ResultsLoaderFactoryImpl.java       |  88 +++++
 .../results/ResultsVerifier.java                |  72 ++++
 .../results/VersionVerifier.java                | 103 +++++
 .../corepersistence/util/CpEntityMapUtils.java  | 323 ++++++++++++++++
 .../corepersistence/util/CpNamingUtils.java     | 106 ++++++
 .../usergrid/persistence/EntityManager.java     |   5 +
 .../cassandra/EntityManagerImpl.java            |   7 +
 .../org/apache/usergrid/CoreApplication.java    |   1 +
 .../CpEntityDeleteListenerTest.java             |   2 +-
 .../CpEntityIndexDeleteListenerTest.java        |   3 +-
 .../corepersistence/CpEntityMapUtilsTest.java   |   2 +
 .../corepersistence/StaleIndexCleanupTest.java  |   1 +
 .../collection/EntityCollectionManager.java     |  27 +-
 .../persistence/collection/EntitySet.java       |  47 +++
 .../persistence/collection/MvccEntity.java      |  68 ++++
 .../persistence/collection/MvccLogEntry.java    |  94 +++++
 .../persistence/collection/VersionSet.java      |  67 ++++
 .../exception/CollectionRuntimeException.java   |   2 +-
 .../exception/WriteCommitException.java         |   4 +-
 .../WriteOptimisticVerifyException.java         |   4 +-
 .../exception/WriteStartException.java          |   4 +-
 .../exception/WriteUniqueVerifyException.java   |   2 +-
 .../collection/guice/CollectionModule.java      |   2 +-
 .../impl/EntityCollectionManagerImpl.java       | 265 ++++++++-----
 .../impl/EntityVersionCleanupTask.java          |   2 +-
 .../mvcc/MvccEntitySerializationStrategy.java   |  20 +-
 .../mvcc/MvccLogEntrySerializationStrategy.java |  17 +-
 .../collection/mvcc/changelog/ChangeLog.java    |  68 ++++
 .../mvcc/changelog/ChangeLogEntry.java          | 104 -----
 .../mvcc/changelog/ChangeLogGenerator.java      |  16 +-
 .../mvcc/changelog/ChangeLogGeneratorImpl.java  | 138 +++----
 .../mvcc/changelog/ChangeLogImpl.java           | 135 +++++++
 .../collection/mvcc/entity/MvccEntity.java      |  68 ----
 .../collection/mvcc/entity/MvccLogEntry.java    |  93 -----
 .../mvcc/entity/MvccValidationUtils.java        |   1 +
 .../mvcc/entity/impl/MvccEntityDeleteEvent.java |   2 +-
 .../mvcc/entity/impl/MvccEntityEvent.java       |   2 +-
 .../mvcc/entity/impl/MvccEntityImpl.java        |   2 +-
 .../mvcc/entity/impl/MvccEntityWriteEvent.java  |   3 +-
 .../mvcc/entity/impl/MvccLogEntryImpl.java      |   2 +-
 .../mvcc/stage/EntityUpdateEvent.java           |   1 -
 .../mvcc/stage/delete/MarkCommit.java           |   4 +-
 .../collection/mvcc/stage/delete/MarkStart.java |   4 +-
 .../collection/mvcc/stage/load/GetVersion.java  |  74 ----
 .../collection/mvcc/stage/load/Load.java        |  98 -----
 .../mvcc/stage/write/RollbackAction.java        |   2 +-
 .../mvcc/stage/write/WriteCommit.java           |   4 +-
 .../mvcc/stage/write/WriteOptimisticVerify.java |  44 +--
 .../collection/mvcc/stage/write/WriteStart.java |   5 +-
 .../mvcc/stage/write/WriteUniqueVerify.java     |   2 +-
 .../collection/serialization/EntityRepair.java  |  38 ++
 .../serialization/OptimisticUpdate.java         |   2 +-
 .../serialization/SerializationFig.java         |  19 +
 .../serialization/impl/EntityRepairImpl.java    | 149 ++++++++
 .../serialization/impl/EntitySetImpl.java       |  62 +++
 .../serialization/impl/LogEntryIterator.java    |   2 +-
 .../MvccEntitySerializationStrategyImpl.java    | 174 ++++++---
 .../MvccLogEntrySerializationStrategyImpl.java  | 129 +++++--
 .../serialization/impl/VersionSetImpl.java      |  80 ++++
 .../persistence/collection/util/RepairUtil.java | 139 -------
 .../collection/EntityCollectionManagerIT.java   | 310 ++++++++++++---
 .../changelog/ChangeLogGeneratorImplTest.java   | 375 +++++++++++--------
 .../mvcc/entity/impl/MvccEntityImplTest.java    |   2 +-
 .../mvcc/entity/impl/MvccLogEntryImplTest.java  |   2 +-
 .../mvcc/stage/AbstractMvccEntityStageTest.java |   2 +-
 .../mvcc/stage/TestEntityGenerator.java         |   2 +-
 .../mvcc/stage/delete/MarkCommitTest.java       |   6 +-
 .../mvcc/stage/delete/MarkStartTest.java        |   4 +-
 .../collection/mvcc/stage/load/LoadTest.java    | 361 ------------------
 .../mvcc/stage/write/WriteCommitTest.java       |   4 +-
 .../stage/write/WriteOptimisticVerifyTest.java  |   4 +-
 .../mvcc/stage/write/WriteStartTest.java        |   4 +-
 .../mvcc/stage/write/WriteUniqueVerifyTest.java |   2 +-
 .../serialization/EntityRepairImplTest.java     | 147 ++++++++
 .../impl/LogEntryIteratorTest.java              |   2 +-
 ...MvccEntitySerializationStrategyImplTest.java |  40 +-
 .../impl/MvccLESSTransientTest.java             |   7 +-
 ...ccLogEntrySerializationStrategyImplTest.java |  19 +-
 .../util/InvalidMvccEntityGenerator.java        |   2 +-
 .../util/InvalidValueGeneratorTest.java         |   2 +-
 .../collection/util/LogEntryMock.java           |   4 +-
 .../usergrid/persistence/index/EntityIndex.java |   6 +
 .../index/impl/EsEntityIndexImpl.java           | 216 ++++++-----
 .../persistence/index/impl/EntityIndexTest.java |   5 +
 .../cassandra/ManagementServiceImpl.java        |  38 +-
 98 files changed, 3730 insertions(+), 2332 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/140e5f79/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 724aa80,de68c24..548afda
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -2175,12 -2191,29 +2187,31 @@@ public class CpEntityManager implement
              String entityType, String propertyName, Object propertyValue )
              throws Exception {
  
-         Results results= this.searchCollection(
-                 getApplication(), 
-                 Schema.defaultCollectionName( entityType), 
-                 Query.searchForProperty(propertyName, propertyValue));
  
-         return results.isEmpty();
+ 
+         return getIdForUniqueEntityField( entityType, propertyName, propertyValue ) == null;
+     }
+ 
+ 
+     /**
+      * Load the unique property for the field
+      */
+     private Id getIdForUniqueEntityField( final String collectionName, final String propertyName,
+                                           final Object propertyValue ) {
 -        CollectionScope collectionScope =
 -                new CollectionScopeImpl( applicationScope.getApplication(), applicationScope.getApplication(),
 -                        CpNamingUtils.getCollectionScopeNameFromEntityType( collectionName
) );
++
++        CollectionScope collectionScope = new CollectionScopeImpl( 
++            applicationScope.getApplication(), 
++            applicationScope.getApplication(),
++            CpNamingUtils.getCollectionScopeNameFromEntityType( collectionName ) );
+ 
+ 
+         final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope
);
+ 
+         //convert to a string, that's what we store
 -        final Id results = ecm.getIdField( new StringField( propertyName, propertyValue.toString()
) ).toBlocking()
 -                              .lastOrDefault( null );
++        final Id results = ecm.getIdField( 
++            new StringField( propertyName, propertyValue.toString() ) ).toBlocking() .lastOrDefault(
null );
+ 
+         return results;
      }
  
  

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/140e5f79/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 02c2c24,72763e8..cf19050
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -21,8 -21,12 +21,11 @@@ import com.google.common.cache.LoadingC
  import com.google.inject.Injector;
  import com.yammer.metrics.annotation.Metered;
  import static java.lang.String.CASE_INSENSITIVE_ORDER;
+ 
+ import java.util.Arrays;
 -import java.util.Collection;
  import java.util.HashMap;
  import java.util.Iterator;
+ import java.util.List;
  import java.util.Map;
  import java.util.TreeMap;
  import java.util.UUID;
@@@ -554,22 -609,44 +608,46 @@@ public class CpEntityManagerFactory imp
      }
  
  
+     /**
+      * TODO, these 3 methods are super janky.  During refactoring we should clean this model
up
+      */
      public void refreshIndex() {
  
-         // refresh special indexes without calling EntityManager refresh because stack overflow

-        
+         // refresh special indexes without calling EntityManager refresh because stack overflow
+         maybeCreateIndexes();
          // system app
  
-         managerCache.getEntityIndex( new ApplicationScopeImpl( 
-                 new SimpleId( SYSTEM_APP_ID, "application" ) ) ).refresh();
+         for ( EntityIndex index : getManagementIndexes() ) {
+             index.refresh();
+         }
+     }
  
-         // default app
-         managerCache.getEntityIndex( new ApplicationScopeImpl( 
-                 new SimpleId( getManagementAppId(), "application" ) ) ).refresh();
  
-         // management app
-         managerCache.getEntityIndex( new ApplicationScopeImpl( 
-                 new SimpleId( getDefaultAppId(), "application" ) ) ).refresh();
+     private void maybeCreateIndexes() {
+         // system app
+         if ( INIT_SYSTEM.getAndSet( true ) ) {
+             return;
+         }
+ 
+         for ( EntityIndex index : getManagementIndexes() ) {
+             index.initializeIndex();
+         }
+     }
+ 
+ 
+     private List<EntityIndex> getManagementIndexes() {
++
+         return Arrays.asList(
 -                managerCache.getEntityIndex( new ApplicationScopeImpl( new SimpleId( SYSTEM_APP_ID,
"application" ) ) ),
++                managerCache.getEntityIndex( new ApplicationScopeImpl( 
++                        new SimpleId( SYSTEM_APP_ID, "application" ) ) ),
+ 
+                 // default app
 -                managerCache.getEntityIndex(
 -                        new ApplicationScopeImpl( new SimpleId( getManagementAppId(), "application"
) ) ),
++                managerCache.getEntityIndex( new ApplicationScopeImpl( 
++                        new SimpleId( getManagementAppId(), "application" ) ) ),
+ 
+                 // management app
 -                managerCache.getEntityIndex(
 -                        new ApplicationScopeImpl( new SimpleId( getDefaultAppId(), "application"
) ) ) );
++                managerCache.getEntityIndex( new ApplicationScopeImpl( 
++                        new SimpleId( getDefaultAppId(), "application" ) ) ) );
      }
  
  
@@@ -587,10 -664,10 +665,10 @@@
              rebuildApplicationIndexes( appUuid, po );
          }
      }
-    
+ 
  
      @Override
 -    public void rebuildInternalIndexes(ProgressObserver po) throws Exception {
 +    public void rebuildInternalIndexes( ProgressObserver po ) throws Exception {
          rebuildApplicationIndexes(SYSTEM_APP_ID, po);
      }
  

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/140e5f79/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index bcfe215,0f31c83..3d5445c
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -1551,136 -1559,12 +1559,10 @@@ public class CpRelationManager implemen
       */
      private Results buildResults(Query query, CandidateResults crs, String collName ) {
  
 -
          logger.debug("buildResults() for {} from {} candidates", collName, crs.size());
  
-         Results results = null;
--
-         EntityIndex index = managerCache.getEntityIndex(applicationScope);
-         EntityIndexBatch indexBatch = index.createBatch();
- 
- 
-         // map of the latest versions, we will discard stale indexes
-         Map<Id, CandidateResult> latestVersions = new LinkedHashMap<Id, CandidateResult>();
- 
-         Iterator<CandidateResult> iter = crs.iterator();
-         while ( iter.hasNext() ) {
- 
-             CandidateResult cr = iter.next();
- 
-             CollectionScope collScope = new CollectionScopeImpl( 
-                 applicationScope.getApplication(), 
-                 applicationScope.getApplication(), 
-                 CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType()
));
- 
-             EntityCollectionManager ecm = managerCache.getEntityCollectionManager(collScope);
- 
-             UUID latestVersion = ecm.getLatestVersion( cr.getId() ).toBlocking().lastOrDefault(null);
- 
-             if ( logger.isDebugEnabled() ) {
-                 logger.debug("Getting version for entity {} from scope\n   app {}\n   owner
{}\n   name {}", 
-                 new Object[] { 
-                     cr.getId(),
-                     collScope.getApplication(), 
-                     collScope.getOwner(), 
-                     collScope.getName() 
-                 });
-             }
- 
-             if ( latestVersion == null ) {
-                 logger.error("Version for Entity {}:{} not found", 
-                         cr.getId().getType(), cr.getId().getUuid());
-                 continue;
-             }
- 
-             if ( cr.getVersion().compareTo( latestVersion) < 0 )  {
-                 logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest
v:{}", 
-                     new Object[] { cr.getId().getUuid(), cr.getId().getType(), 
-                         cr.getVersion(), latestVersion});
- 
-                 IndexScope indexScope = new IndexScopeImpl(
-                     cpHeadEntity.getId(),
-                     CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType()
));
-                 indexBatch.deindex( indexScope, cr);
- 
-                 continue;
-             }
- 
-             CandidateResult alreadySeen = latestVersions.get( cr.getId() ); 
- 
-             if ( alreadySeen == null ) { // never seen it, so add to map
-                 latestVersions.put( cr.getId(), cr );
- 
-             } else {
-                 // we seen this id before, only add entity if we now have newer version
-                 if ( latestVersion.compareTo( alreadySeen.getVersion() ) > 0 ) {
- 
-                     latestVersions.put( cr.getId(), cr);
- 
-                     IndexScope indexScope = new IndexScopeImpl(
-                         cpHeadEntity.getId(),
-                         CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType()
));
-                     indexBatch.deindex( indexScope, alreadySeen);
-                 }
-             }
-         }
- 
-         indexBatch.execute();
- 
-         if (query.getLevel().equals(Level.IDS)) {
- 
-             List<UUID> ids = new ArrayList<UUID>();
-             for ( Id id : latestVersions.keySet() ) {
-                 CandidateResult cr = latestVersions.get(id);
-                 ids.add( cr.getId().getUuid() );
-             }
-             results = Results.fromIdList(ids);
- 
-         } else if (query.getLevel().equals(Level.REFS)) {
- 
-             List<EntityRef> refs = new ArrayList<EntityRef>();
-             for ( Id id : latestVersions.keySet() ) {
-                 CandidateResult cr = latestVersions.get(id);
-                 refs.add( new SimpleEntityRef( cr.getId().getType(), cr.getId().getUuid()));
-             }
-             results = Results.fromRefList( refs );
- 
-         } else {
- 
-             List<Entity> entities = new ArrayList<Entity>();
-             for (Id id : latestVersions.keySet()) {
- 
-                 CandidateResult cr = latestVersions.get(id);
- 
-                 CollectionScope collScope = new CollectionScopeImpl( 
-                     applicationScope.getApplication(), 
-                     applicationScope.getApplication(), 
-                     CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType()
));
- 
-                 EntityCollectionManager ecm = managerCache.getEntityCollectionManager(collScope);
- 
-                 org.apache.usergrid.persistence.model.entity.Entity e = 
-                         ecm.load( cr.getId() ).toBlocking().lastOrDefault(null);
- 
-                 if ( e == null ) {
-                     logger.error("Entity {}:{} not found", 
-                             cr.getId().getType(), cr.getId().getUuid());
-                     continue;
-                 }
- 
-                 Entity entity = EntityFactory.newEntity(
-                         e.getId().getUuid(), e.getField("type").getValue().toString());
- 
-                 Map<String, Object> entityMap = CpEntityMapUtils.toMap(e);
-                 entity.addProperties(entityMap);
-                 entities.add(entity);
-             }
- 
-             if (entities.size() == 1) {
-                 results = Results.fromEntity(entities.get(0));
-             } else {
-                 results = Results.fromEntities(entities);
-             }
-         }
 -        final Results results = this.resultsLoaderFactory.getLoader( applicationScope, this.headEntity,
query.getResultsLevel() ).getResults( crs );
 -
++        final Results results = this.resultsLoaderFactory.getLoader( 
++                applicationScope, this.headEntity, query.getResultsLevel() ).getResults(
crs );
  
          results.setCursor( crs.getCursor() );
          results.setQueryProcessor( new CpQueryProcessor(em, query, headEntity, collName)
);

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/140e5f79/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/140e5f79/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------


Mime
View raw message