usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mdun...@apache.org
Subject [2/3] usergrid git commit: Collection clear using version changes
Date Thu, 17 Aug 2017 16:01:51 GMT
http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
index f47afe6..d9ee87e 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
@@ -45,6 +45,7 @@ import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.CollectionUtils;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 import org.junit.Before;
 import org.junit.Test;
@@ -205,7 +206,9 @@ public class IndexServiceTest {
 
         assertEquals( 1, collectionResults.size() );
 
-        assertEquals( testEntity.getId(), collectionResults.get( 0 ).getId() );
+        // with collection versioning, empty versions are included
+        assertEquals(testEntity.getId().getType(), CollectionUtils.stripEmptyVersion(collectionResults.get(0).getId().getType()));
+        assertEquals(testEntity.getId().getUuid(), collectionResults.get(0).getId().getUuid());
 
 
         //query until the connection edge is available
@@ -216,7 +219,9 @@ public class IndexServiceTest {
 
         assertEquals( 1, connectionResults.size() );
 
-        assertEquals( testEntity.getId(), connectionResults.get( 0 ).getId() );
+        // with collection versioning, empty versions are included
+        assertEquals(testEntity.getId().getType(), CollectionUtils.stripEmptyVersion(collectionResults.get(0).getId().getType()));
+        assertEquals(testEntity.getId().getUuid(), collectionResults.get(0).getId().getUuid());
     }
 
     /**
@@ -294,8 +299,9 @@ public class IndexServiceTest {
 
         assertEquals( 1, collectionResults.size() );
 
-        assertEquals( testEntity.getId(), collectionResults.get( 0 ).getId() );
-
+        // with collection versioning, empty versions are included
+        assertEquals(testEntity.getId().getType(), CollectionUtils.stripEmptyVersion(collectionResults.get(0).getId().getType()));
+        assertEquals(testEntity.getId().getUuid(), collectionResults.get(0).getId().getUuid());
 
         final SearchEdge connectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( connectionSearch );
 
@@ -306,7 +312,8 @@ public class IndexServiceTest {
 
         assertEquals( 1, connectionResults.size() );
 
-        assertEquals( testEntity.getId(), connectionResults.get( 0 ).getId() );
+        assertEquals(testEntity.getId().getType(), CollectionUtils.stripEmptyVersion(collectionResults.get(0).getId().getType()));
+        assertEquals(testEntity.getId().getUuid(), collectionResults.get(0).getId().getUuid());
 
 
         final SearchEdge lastConnectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( lastSearch );
@@ -318,7 +325,8 @@ public class IndexServiceTest {
 
         assertEquals( 1, lastConnectionResults.size() );
 
-        assertEquals( testEntity.getId(), lastConnectionResults.get( 0 ).getId() );
+        assertEquals(testEntity.getId().getType(), CollectionUtils.stripEmptyVersion(lastConnectionResults.get(0).getId().getType()));
+        assertEquals(testEntity.getId().getUuid(), lastConnectionResults.get(0).getId().getUuid());
     }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
index 281f2af..e9d9e62 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
@@ -163,7 +163,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
 
         waitForRebuild( status, reIndexService );
 
-        app.waitForQueueDrainAndRefreshIndex(5000);
+        app.waitForQueueDrainAndRefreshIndex(10000);
 
         // ----------------- test that we can read the catherder collection and not the catshepard
 
@@ -172,7 +172,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
     }
 
 
-    @Test( timeout = 120000 )
+    @Test( timeout = 240000 )
     public void rebuildIndex() throws Exception {
 
         logger.info( "Started rebuildIndex()" );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index d22ac65..9b7566a 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -147,7 +147,7 @@ public class GraphManagerImpl implements GraphManager {
     public Observable<MarkedEdge> writeEdge( final Edge edge ) {
         GraphValidation.validateEdge( edge );
 
-        final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, false );
+        final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, false, false, false );
 
         final Observable<MarkedEdge> observable = Observable.just( markedEdge ).map( edge1 -> {
 
@@ -178,7 +178,7 @@ public class GraphManagerImpl implements GraphManager {
     public Observable<MarkedEdge> markEdge( final Edge edge ) {
         GraphValidation.validateEdge( edge );
 
-        final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, true );
+        final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, true, false, false );
 
         final Observable<MarkedEdge> observable = Observable.just( markedEdge ).map( edge1 -> {
 
@@ -269,6 +269,7 @@ public class GraphManagerImpl implements GraphManager {
         final Observable<MarkedEdge> nodeObservable =
             Observable.just( inputNode )
                 .map( node -> nodeSerialization.getMaxVersion( scope, node ) )
+                //.doOnNext(maxTimestamp -> logger.info("compactNode maxTimestamp={}", maxTimestamp.toString()))
                 .takeWhile(maxTimestamp -> maxTimestamp.isPresent() )
                 //map our delete listener
                 .flatMap( timestamp -> nodeDeleteListener.receive( scope, inputNode, startTime ) );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleEdge.java
index a89cd96..23ab074 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleEdge.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.persistence.graph.impl;
 import org.apache.usergrid.persistence.graph.Edge;
 import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
 import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
 
 
 /**
@@ -54,6 +55,15 @@ public class SimpleEdge implements Edge {
         GraphValidation.validateEdge( this );
     }
 
+    public SimpleEdge( final Edge another, boolean includeEmptyVersion ) {
+        this(
+            new SimpleId(another.getSourceNode(), includeEmptyVersion),
+            another.getType(),
+            new SimpleId(another.getTargetNode(), includeEmptyVersion),
+            another.getTimestamp()
+        );
+    }
+
 
     @Override
     public Id getSourceNode() {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
index 9c35e2e..c244ad4 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
@@ -25,6 +25,7 @@ import org.apache.usergrid.persistence.graph.MarkedEdge;
 import org.apache.usergrid.persistence.model.entity.Id;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
 
 
 /**
@@ -63,8 +64,22 @@ public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge {
     }
 
 
-    public SimpleMarkedEdge( final Edge edge, final boolean isDeleted ) {
-        this( edge.getSourceNode(), edge.getType(), edge.getTargetNode(), edge.getTimestamp(), isDeleted );
+    public SimpleMarkedEdge( final Edge edge, final boolean isDeleted, final boolean isSourceNodeDeleted, final boolean isTargetNodeDeleted ) {
+        this(edge.getSourceNode(), edge.getType(), edge.getTargetNode(), edge.getTimestamp(),
+            isDeleted, isSourceNodeDeleted, isTargetNodeDeleted);
+    }
+
+
+    public SimpleMarkedEdge( final MarkedEdge another, final boolean includeEmptyVersion ) {
+        this(
+            new SimpleId(another.getSourceNode(), includeEmptyVersion),
+            another.getType(),
+            new SimpleId(another.getTargetNode(), includeEmptyVersion),
+            another.getTimestamp(),
+            another.isDeleted(),
+            another.isSourceNodeDelete(),
+            another.isTargetNodeDeleted()
+        );
     }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
index b915e4f..707b45f 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
@@ -76,6 +76,12 @@ public class Entity extends EntityObject {
         this.version = version;
     }
 
+    public Entity(final Entity another, boolean includeEmptyVersion) {
+        this(new SimpleId(another.getId(), includeEmptyVersion), another.getVersion());
+        this.setFieldMap(another.getFieldMap());
+        this.setSize(another.getSize());
+    }
+
 
     /**
      * Generate a new entity with the given type and a new id

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java
index 39014ab..6e048ac 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java
@@ -44,6 +44,12 @@ public interface Id extends Comparable<Id>, Serializable {
      */
     String getType();
 
+    /**
+     * Get the unique type for this id, using specified empty version handling
+     * @return
+     */
+    String getType(boolean includeEmptyVersion);
+
 
     //Application -> Class "Application"
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
index 6a45558..11bf0e4 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/SimpleId.java
@@ -29,6 +29,8 @@ import org.apache.usergrid.persistence.model.util.Verify;
 import com.fasterxml.uuid.UUIDComparator;
 import com.google.common.base.Preconditions;
 
+import org.apache.usergrid.persistence.model.util.CollectionUtils;
+
 
 /** @author tnine */
 public class SimpleId implements Id, Serializable {
@@ -56,6 +58,11 @@ public class SimpleId implements Id, Serializable {
         this.type = type;
     }
 
+    public SimpleId(final Id another, boolean includeEmptyVersion) {
+        this.uuid = another.getUuid();
+        this.type = another.getType(includeEmptyVersion);
+    }
+
 
     /**
      * Create a new ID.  Should only be used for new entities
@@ -78,6 +85,18 @@ public class SimpleId implements Id, Serializable {
     }
 
 
+    @Override
+    public String getType(boolean includeEmptyVersion) {
+        String retType;
+        if (includeEmptyVersion) {
+            retType = CollectionUtils.addEmptyVersion(type);
+        } else {
+            retType = CollectionUtils.stripEmptyVersion(type);
+        }
+        return retType;
+    }
+
+
 
     /**
      * Do not delete!  Needed for Jackson

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/CollectionUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/CollectionUtils.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/CollectionUtils.java
new file mode 100644
index 0000000..bff9509
--- /dev/null
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/CollectionUtils.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.persistence.model.util;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class CollectionUtils {
+    public static final String VERSIONED_NAME_SEPARATOR = "~-_~_-~";
+
+    public static final String COLLECTION_USERS = "users";
+    public static final String COLLECTION_GROUPS = "groups";
+    public static final String COLLECTION_ASSETS = "assets";
+    public static final String COLLECTION_ACTIVITIES = "activities";
+    public static final String COLLECTION_EVENTS = "events";
+    public static final String COLLECTION_FOLDERS = "folders";
+    public static final String COLLECTION_DEVICES = "devices";
+    public static final String COLLECTION_NOTIFICATIONS = "notifications";
+    public static final String COLLECTION_ROLES = "roles";
+
+    public static final String COLLECTION_ENTITY_USER = "user";
+    public static final String COLLECTION_ENTITY_GROUP = "group";
+    public static final String COLLECTION_ENTITY_ASSET = "asset";
+    public static final String COLLECTION_ENTITY_ACTIVITY = "activity";
+    public static final String COLLECTION_ENTITY_EVENT = "event";
+    public static final String COLLECTION_ENTITY_FOLDER = "folder";
+    public static final String COLLECTION_ENTITY_DEVICE = "device";
+    public static final String COLLECTION_ENTITY_NOTIFICATION = "notification";
+    public static final String COLLECTION_ENTITY_ROLE = "role";
+
+    private static final Set<String> customNames;
+
+    static {
+        customNames = new HashSet<>();
+        customNames.add(COLLECTION_USERS);
+        customNames.add(COLLECTION_GROUPS);
+        customNames.add(COLLECTION_ASSETS);
+        customNames.add(COLLECTION_ACTIVITIES);
+        customNames.add(COLLECTION_EVENTS);
+        customNames.add(COLLECTION_FOLDERS);
+        customNames.add(COLLECTION_DEVICES);
+        customNames.add(COLLECTION_NOTIFICATIONS);
+        customNames.add(COLLECTION_ROLES);
+
+        customNames.add(COLLECTION_ENTITY_USER);
+        customNames.add(COLLECTION_ENTITY_GROUP);
+        customNames.add(COLLECTION_ENTITY_ASSET);
+        customNames.add(COLLECTION_ENTITY_ACTIVITY);
+        customNames.add(COLLECTION_ENTITY_EVENT);
+        customNames.add(COLLECTION_ENTITY_FOLDER);
+        customNames.add(COLLECTION_ENTITY_DEVICE);
+        customNames.add(COLLECTION_ENTITY_NOTIFICATION);
+        customNames.add(COLLECTION_ENTITY_ROLE);
+    }
+
+    public static boolean isCustomCollectionOrEntityName(String collectionName) {
+        return !customNames.contains(collectionName);
+    }
+
+    public static String stripEmptyVersion(final String name) {
+        // versioned name with empty version is name followed by separator
+        if (name.endsWith(VERSIONED_NAME_SEPARATOR)) {
+            return name.substring(0, name.length() - VERSIONED_NAME_SEPARATOR.length());
+        }
+        return name;
+    }
+
+    public static String addEmptyVersion(final String name) {
+        if (!isCustomCollectionOrEntityName(name) ||
+            name.contains(VERSIONED_NAME_SEPARATOR)) {
+            // not custom collection or already has version
+            return name;
+        }
+        return name + VERSIONED_NAME_SEPARATOR;
+    }
+
+    public static String handleEmptyVersion(final String name, boolean addEmptyVersion) {
+        String ret;
+        if (addEmptyVersion) {
+            ret = addEmptyVersion(name);
+        } else {
+            ret = stripEmptyVersion(name);
+        }
+        return ret;
+    }
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
index aefceda..b5477f8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
@@ -33,6 +33,8 @@ import org.apache.usergrid.persistence.model.entity.Id;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
 
@@ -43,6 +45,8 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createInd
 @JsonTypeInfo( use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "@class" )
 public class DeIndexOperation implements BatchOperation {
 
+    private static final Logger logger = LoggerFactory.getLogger( DeIndexOperation.class );
+
     @JsonProperty
     public String[] indexes;
 
@@ -59,11 +63,13 @@ public class DeIndexOperation implements BatchOperation {
                              UUID version ) {
         this.indexes = indexes;
         this.documentId = createIndexDocId( applicationScope, id, version, searchEdge );
+        //logger.info("documentId={}", this.documentId);
     }
 
     public DeIndexOperation( String[] indexes, String docId) {
         this.indexes = indexes;
         this.documentId = docId;
+        //logger.info("documentId={}", this.documentId);
     }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 1bb97b3..ac2f50a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.persistence.index.impl;
 import java.util.Set;
 import java.util.UUID;
 
+import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -88,7 +89,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
         }
 
         //add app id for indexing
-        container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, indexEdge, entity,fieldsToIndex));
+        container.addIndexRequest(new IndexOperation(writeAlias, applicationScope,
+            new IndexEdgeImpl(indexEdge, false),
+            new Entity(entity, false),fieldsToIndex));
         return this;
     }
 
@@ -111,7 +114,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
         }
 
 
-        container.addDeIndexRequest(new DeIndexOperation(indexes, applicationScope, searchEdge, id, version));
+        container.addDeIndexRequest(new DeIndexOperation(indexes, applicationScope,
+            new SearchEdgeImpl(searchEdge, false),
+            new SimpleId(id, false), version));
 
         return this;
     }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java
index 5b525cb..409639d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexEdgeImpl.java
@@ -26,6 +26,7 @@ package org.apache.usergrid.persistence.index.impl;
 
 import org.apache.usergrid.persistence.index.IndexEdge;
 import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
 
 
 /**
@@ -41,6 +42,15 @@ public class IndexEdgeImpl extends SearchEdgeImpl implements IndexEdge {
         this.timestamp = timestamp;
     }
 
+    public IndexEdgeImpl( final IndexEdge another, final boolean includeEmptyVersion) {
+        this(
+            new SimpleId(another.getNodeId(), includeEmptyVersion),
+            another.getEdgeName(),
+            another.getNodeType(),
+            another.getTimestamp()
+        );
+    }
+
 
     @Override
     public long getTimestamp() {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
index f8560ba..f29e7bc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexOperation.java
@@ -33,6 +33,8 @@ import org.elasticsearch.client.Client;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Optional;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**
@@ -40,6 +42,8 @@ import com.google.common.base.Optional;
  */
 public class IndexOperation implements BatchOperation {
 
+    private static final Logger logger = LoggerFactory.getLogger( IndexOperation.class );
+
     @JsonProperty
     public String writeAlias;
     @JsonProperty
@@ -61,6 +65,7 @@ public class IndexOperation implements BatchOperation {
         this.writeAlias = writeAlias;
         this.data = data;
         this.documentId = documentId;
+        //logger.info("documentId={}", documentId);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 5fe5b39..c878ea2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -1,4 +1,4 @@
-package org.apache.usergrid.persistence.index.impl;/*
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -17,12 +17,14 @@ package org.apache.usergrid.persistence.index.impl;/*
  * under the License.
  */
 
+package org.apache.usergrid.persistence.index.impl;
 
 import java.util.UUID;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.util.CollectionUtils;
 import org.apache.usergrid.persistence.index.CandidateResult;
 import org.apache.usergrid.persistence.index.GeoCandidateResult;
 import org.apache.usergrid.persistence.index.IndexEdge;
@@ -69,9 +71,9 @@ public class IndexingUtils {
     private static final Pattern DOCUMENT_PATTERN = Pattern.compile( DOCUMENT_ID_REGEX );
 
     // These are not allowed in document type names: _ . , | #
-    public static final String FIELD_SEPERATOR = ".";
+    public static final String FIELD_SEPARATOR = ".";
 
-    public static final String ID_SEPERATOR = ",";
+    public static final String ID_SEPARATOR = ",";
 
 
     /**
@@ -137,13 +139,17 @@ public class IndexingUtils {
      *
      * TODO make this format more readable and parsable
      */
-    public static String createContextName( final ApplicationScope applicationScope, final SearchEdge scope ) {
+    public static String createContextName( final ApplicationScope applicationScope, final SearchEdge searchEdge ) {
+        SearchEdge strippedSearchEdge = new SearchEdgeImpl(
+            new SimpleId(searchEdge.getNodeId().getUuid(), CollectionUtils.stripEmptyVersion(searchEdge.getNodeId().getType())),
+            searchEdge.getEdgeName(), searchEdge.getNodeType()
+        );
         StringBuilder sb = new StringBuilder();
         idString( sb, APPID_NAME, applicationScope.getApplication() );
-        sb.append( FIELD_SEPERATOR );
-        idString( sb, NODEID_NAME, scope.getNodeId() );
-        sb.append( FIELD_SEPERATOR );
-        appendField( sb, EDGE_NAME, scope.getEdgeName() );
+        sb.append(FIELD_SEPARATOR);
+        idString( sb, NODEID_NAME, strippedSearchEdge.getNodeId() );
+        sb.append(FIELD_SEPARATOR);
+        appendField( sb, EDGE_NAME, strippedSearchEdge.getEdgeName() );
         return sb.toString();
     }
 
@@ -163,34 +169,41 @@ public class IndexingUtils {
     public static String createIndexDocId( final ApplicationScope applicationScope, final Id entityId,
                                            final UUID version, final SearchEdge searchEdge ) {
 
+        // strip empty collection versions to maintain backward compatibility
+        Id strippedEntityId = new SimpleId(entityId.getUuid(), CollectionUtils.stripEmptyVersion(entityId.getType()));
+        SearchEdge strippedSearchEdge = new SearchEdgeImpl(
+            new SimpleId(searchEdge.getNodeId().getUuid(), CollectionUtils.stripEmptyVersion(searchEdge.getNodeId().getType())),
+            searchEdge.getEdgeName(), searchEdge.getNodeType()
+        );
+
         StringBuilder sb = new StringBuilder();
         idString( sb, APPID_NAME, applicationScope.getApplication() );
-        sb.append( FIELD_SEPERATOR );
-        idString( sb, ENTITY_ID_FIELDNAME, entityId );
-        sb.append( FIELD_SEPERATOR );
+        sb.append(FIELD_SEPARATOR);
+        idString( sb, ENTITY_ID_FIELDNAME, strippedEntityId );
+        sb.append(FIELD_SEPARATOR);
         appendField( sb, VERSION_NAME, version.toString() );
-        sb.append( FIELD_SEPERATOR );
-        idString( sb, NODEID_NAME, searchEdge.getNodeId() );
-        sb.append( FIELD_SEPERATOR );
-        appendField( sb, EDGE_NAME, searchEdge.getEdgeName() );
-        sb.append( FIELD_SEPERATOR );
-        appendField( sb, NODE_TYPE_NAME, searchEdge.getNodeType().name() );
+        sb.append(FIELD_SEPARATOR);
+        idString( sb, NODEID_NAME, strippedSearchEdge.getNodeId() );
+        sb.append(FIELD_SEPARATOR);
+        appendField( sb, EDGE_NAME, strippedSearchEdge.getEdgeName() );
+        sb.append(FIELD_SEPARATOR);
+        appendField( sb, NODE_TYPE_NAME, strippedSearchEdge.getNodeType().name() );
 
         return sb.toString();
     }
 
 
-    public static final String entityId( final Id id ) {
+    public static String entityId( final Id id ) {
         return idString( ENTITY_NAME, id );
     }
 
 
-    public static final String applicationId( final Id id ) {
+    public static String applicationId( final Id id ) {
         return idString( APPID_NAME, id );
     }
 
 
-    public static final String nodeId( final Id id ) {
+    public static String nodeId( final Id id ) {
         return idString( NODEID_NAME, id );
     }
 
@@ -198,7 +211,7 @@ public class IndexingUtils {
     /**
      * Construct and Id string with the specified type for the id provided.
      */
-    private static final String idString( final String type, final Id id ) {
+    private static String idString( final String type, final Id id ) {
         final StringBuilder stringBuilder = new StringBuilder();
 
         idString( stringBuilder, type, id );
@@ -211,7 +224,7 @@ public class IndexingUtils {
      * Append the id to the string
      */
     private static final void idString( final StringBuilder builder, final String type, final Id id ) {
-        builder.append( type ).append( "(" ).append( id.getUuid() ).append( ID_SEPERATOR )
+        builder.append( type ).append( "(" ).append( id.getUuid() ).append(ID_SEPARATOR)
                .append( id.getType().toLowerCase() ).append( ")" );
     }
 
@@ -254,7 +267,7 @@ public class IndexingUtils {
 
         //Other fields can be parsed using groups.  The groups start at value 1, group 0 is the entire match
         final String entityUUID = matcher.group(3);
-        final String entityType = matcher.group(4);
+        final String entityType = CollectionUtils.addEmptyVersion(matcher.group(4));
 
         final String versionUUID = matcher.group(5);
 
@@ -297,8 +310,8 @@ public class IndexingUtils {
         StringBuilder sb = new StringBuilder();
 
         idString( sb, APPID_NAME, applicationScope.getApplication() );
-        sb.append( FIELD_SEPERATOR );
-        sb.append( ENTITY_TYPE_NAME).append("(" ).append( type ).append( ")" );
+        sb.append(FIELD_SEPARATOR);
+        sb.append( ENTITY_TYPE_NAME).append("(" ).append( CollectionUtils.stripEmptyVersion(type) ).append( ")" );
         return sb.toString();
     }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java
index 3b62691..7bb2db5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchEdgeImpl.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.persistence.index.impl;
 
 import org.apache.usergrid.persistence.index.SearchEdge;
 import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
 
 
 /**
@@ -60,6 +61,14 @@ public class SearchEdgeImpl implements SearchEdge {
         this.nodeType = nodeType;
     }
 
+    public SearchEdgeImpl( final SearchEdge another, boolean includeEmptyVersion) {
+        this(
+            new SimpleId(another.getNodeId(), includeEmptyVersion),
+            another.getEdgeName(),
+            another.getNodeType()
+        );
+    }
+
 
     @Override
     public Id getNodeId() {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index ac7d10d..1e75cea 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -32,6 +32,7 @@ import com.google.common.base.Optional;
 import org.apache.usergrid.persistence.core.CassandraFig;
 import org.apache.usergrid.persistence.index.*;
 import org.apache.usergrid.persistence.model.field.*;
+import org.apache.usergrid.persistence.model.util.CollectionUtils;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -170,24 +171,29 @@ public class EntityIndexTest extends BaseIT {
         final CandidateResult candidate1 = candidateResults.get(0);
 
         //check the id and version
-        assertEquals( entity1.getId(), candidate1.getId() );
+        // with collection versioning, empty versions are included in results
+        assertEquals(entity1.getId().getType(), CollectionUtils.stripEmptyVersion(candidate1.getId().getType()));
+        assertEquals(entity1.getId().getUuid(), candidate1.getId().getUuid());
         assertEquals(entity1.getVersion(), candidate1.getVersion());
 
 
         final CandidateResult candidate2 = candidateResults.get(1);
 
         //check the id and version
-        assertEquals( entity2.getId(), candidate2.getId() );
+        assertEquals(entity2.getId().getType(), CollectionUtils.stripEmptyVersion(candidate2.getId().getType()));
+        assertEquals(entity2.getId().getUuid(), candidate2.getId().getUuid());
         assertEquals( entity2.getVersion(), candidate2.getVersion() );
 
         //make sure we can query uuids out as strings and not wrapped
         candidateResults =
             entityIndex.search( indexEdge, searchTypes, "select * where testuuid = '"+uuid+"'", 100, 0, false );
-        assertEquals(entity1.getId(),candidateResults.get(0).getId());
+        assertEquals(entity1.getId().getType(), CollectionUtils.stripEmptyVersion(candidateResults.get(0).getId().getType()));
+        assertEquals(entity1.getId().getUuid(), candidateResults.get(0).getId().getUuid());
 
         candidateResults =
             entityIndex.search( indexEdge, searchTypes, "select * where testuuid = "+uuid, 100, 0, false);
-        assertEquals(entity1.getId(),candidateResults.get(0).getId());
+        assertEquals(entity1.getId().getType(), CollectionUtils.stripEmptyVersion(candidateResults.get(0).getId().getType()));
+        assertEquals(entity1.getId().getUuid(), candidateResults.get(0).getId().getUuid());
     }
 
 
@@ -519,7 +525,8 @@ public class EntityIndexTest extends BaseIT {
         final String query = "where username = 'edanuff'";
 
         CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0, false);
-        assertEquals( user.getId(), r.get( 0 ).getId());
+        assertEquals(user.getId().getType(), CollectionUtils.stripEmptyVersion(r.get(0).getId().getType()));
+        assertEquals(user.getId().getUuid(), r.get(0).getId().getUuid());
 
         batch.deindex( indexSCope, user.getId(), user.getVersion() );
         indexProducer.put(batch.build()).subscribe();;
@@ -734,7 +741,10 @@ public class EntityIndexTest extends BaseIT {
 
         final CandidateResults r =
             entityIndex.search( indexSCope, SearchTypes.fromTypes(entityId.getType()), query, 10, 0, false);
-        assertEquals(user.getId(), r.get(0).getId());
+
+        // with collection versioning, empty versions are included in results
+        assertEquals(user.getId().getType(), CollectionUtils.stripEmptyVersion(r.get(0).getId().getType()));
+        assertEquals(user.getId().getUuid(), r.get(0).getId().getUuid());
     }
 
 
@@ -774,7 +784,9 @@ public class EntityIndexTest extends BaseIT {
         final CandidateResults r =
             entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), query, 10, 0, false);
 
-        assertEquals(user.getId(), r.get(0).getId());
+        // with collection versioning, empty versions are included in results
+        assertEquals(user.getId().getType(), CollectionUtils.stripEmptyVersion(r.get(0).getId().getType()));
+        assertEquals(user.getId().getUuid(), r.get(0).getId().getUuid());
 
         //shouldn't match
         final String queryNoWildCard = "where string = 'I am'";
@@ -833,8 +845,11 @@ public class EntityIndexTest extends BaseIT {
             entityIndex.search(indexSCope, SearchTypes.fromTypes( first.getId().getType() ), ascQuery, 10 , 0, false);
 
 
-        assertEquals( first.getId(), ascResults.get( 0).getId() );
-        assertEquals( second.getId(), ascResults.get( 1 ).getId() );
+        // with collection versioning, empty versions are included in results
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(ascResults.get(0).getId().getType()));
+        assertEquals(first.getId().getUuid(), ascResults.get(0).getId().getUuid());
+        assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(ascResults.get(1).getId().getType()));
+        assertEquals(second.getId().getUuid(), ascResults.get(1).getId().getUuid());
 
 
         //search in reversed
@@ -844,8 +859,11 @@ public class EntityIndexTest extends BaseIT {
             entityIndex.search(indexSCope, SearchTypes.fromTypes( first.getId().getType() ), descQuery, 10 , 0, false);
 
 
-        assertEquals( second.getId(), descResults.get( 0).getId() );
-        assertEquals( first.getId(), descResults.get( 1 ).getId() );
+        // with collection versioning, empty versions are included when parsing doc IDs
+        assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(descResults.get(0).getId().getType()));
+        assertEquals(second.getId().getUuid(), descResults.get(0).getId().getUuid());
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(descResults.get(1).getId().getType()));
+        assertEquals(first.getId().getUuid(), descResults.get(1).getId().getUuid());
     }
 
 
@@ -899,7 +917,8 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals(1, singleResults.size());
-        assertEquals(first.getId(), singleResults.get(0).getId());
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(singleResults.get(0).getId().getType()));
+        assertEquals(first.getId().getUuid(), singleResults.get(0).getId().getUuid());
 
 
         //search in reversed
@@ -910,8 +929,11 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 2, singleKeywordUnion.size() );
-        assertEquals( second.getId(), singleKeywordUnion.get( 0).getId() );
-        assertEquals( first.getId(), singleKeywordUnion.get( 1 ).getId() );
+        // with collection versioning, empty versions are included when parsing doc IDs
+        assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(singleKeywordUnion.get(0).getId().getType()));
+        assertEquals(second.getId().getUuid(), singleKeywordUnion.get(0).getId().getUuid());
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(singleKeywordUnion.get(1).getId().getType()));
+        assertEquals(first.getId().getUuid(), singleKeywordUnion.get(1).getId().getUuid());
 
 
         final String twoKeywordMatches = "string contains 'alpha' OR string contains 'long'";
@@ -921,8 +943,10 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 2, towMatchResults.size() );
-        assertEquals(second.getId(), towMatchResults.get( 0).getId() );
-        assertEquals(first.getId(), towMatchResults.get( 1 ).getId() );
+        assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(towMatchResults.get(0).getId().getType()));
+        assertEquals(second.getId().getUuid(), towMatchResults.get(0).getId().getUuid());
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(towMatchResults.get(1).getId().getType()));
+        assertEquals(first.getId().getUuid(), towMatchResults.get(1).getId().getUuid());
     }
 
 
@@ -980,7 +1004,8 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 1, notFirstResults.size() );
-        assertEquals(second.getId(), notFirstResults.get( 0 ).getId() );
+        assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(notFirstResults.get(0).getId().getType()));
+        assertEquals(second.getId().getUuid(), notFirstResults.get(0).getId().getUuid());
 
 
         //search in reversed
@@ -991,7 +1016,8 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 1, notSecondUnion.size() );
-        assertEquals( first.getId(), notSecondUnion.get( 0 ).getId() );
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(notSecondUnion.get(0).getId().getType()));
+        assertEquals(first.getId().getUuid(), notSecondUnion.get(0).getId().getUuid());
 
 
         final String notBothReturn = "NOT int = 3";
@@ -1001,8 +1027,10 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 2, notBothReturnResults.size() );
-        assertEquals( second.getId(), notBothReturnResults.get( 0).getId() );
-        assertEquals( first.getId(), notBothReturnResults.get( 1 ).getId() );
+        assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(notBothReturnResults.get(0).getId().getType()));
+        assertEquals(second.getId().getUuid(), notBothReturnResults.get(0).getId().getUuid());
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(notBothReturnResults.get(1).getId().getType()));
+        assertEquals(first.getId().getUuid(), notBothReturnResults.get(1).getId().getUuid());
 
 
         final String notFilterBoth = "(NOT int = 1) AND (NOT int = 2) ";
@@ -1020,8 +1048,10 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 2, noMatchesAndResults.size() );
-        assertEquals( second.getId(), noMatchesAndResults.get( 0).getId() );
-        assertEquals( first.getId(), noMatchesAndResults.get( 1 ).getId() );
+        assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(noMatchesAndResults.get(0).getId().getType()));
+        assertEquals(second.getId().getUuid(), noMatchesAndResults.get(0).getId().getUuid());
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(noMatchesAndResults.get(1).getId().getType()));
+        assertEquals(first.getId().getUuid(), noMatchesAndResults.get(1).getId().getUuid());
 
 
         final String noMatchesOr = "(NOT int = 3) AND (NOT int = 4)";
@@ -1031,8 +1061,10 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 2, noMatchesOrResults.size() );
-        assertEquals( second.getId(), noMatchesOrResults.get( 0).getId() );
-        assertEquals( first.getId(), noMatchesOrResults.get( 1 ).getId() );
+        assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(noMatchesOrResults.get(0).getId().getType()));
+        assertEquals(second.getId().getUuid(), noMatchesOrResults.get(0).getId().getUuid());
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(noMatchesOrResults.get(1).getId().getType()));
+        assertEquals(first.getId().getUuid(), noMatchesOrResults.get(1).getId().getUuid());
     }
 
 
@@ -1090,7 +1122,9 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 1, notFirstResults.size() );
-        assertEquals(second.getId(), notFirstResults.get( 0 ).getId() );
+        // with collection versioning, empty versions are included
+        assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(notFirstResults.get(0).getId().getType()));
+        assertEquals(second.getId().getUuid(), notFirstResults.get(0).getId().getUuid());
 
 
         final String notFirstWildCard = "NOT string = 'I ate*'";
@@ -1100,7 +1134,8 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 1, notFirstWildCardResults.size() );
-        assertEquals(second.getId(), notFirstWildCardResults.get( 0 ).getId() );
+        assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(notFirstWildCardResults.get(0).getId().getType()));
+        assertEquals(second.getId().getUuid(), notFirstWildCardResults.get(0).getId().getUuid());
 
 
         final String notFirstContains = "NOT string contains 'sammich'";
@@ -1110,7 +1145,8 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 1, notFirstContainsResults.size() );
-        assertEquals(second.getId(), notFirstContainsResults.get( 0 ).getId() );
+        assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(notFirstContainsResults.get(0).getId().getType()));
+        assertEquals(second.getId().getUuid(), notFirstContainsResults.get(0).getId().getUuid());
 
 
         //search in reversed
@@ -1121,7 +1157,8 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 1, notSecondUnion.size() );
-        assertEquals( first.getId(), notSecondUnion.get( 0 ).getId() );
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(notSecondUnion.get(0).getId().getType()));
+        assertEquals(first.getId().getUuid(), notSecondUnion.get(0).getId().getUuid());
 
 
         final String notSecondWildcard = "NOT string = 'I drank*'";
@@ -1131,7 +1168,8 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 1, notSecondWildcardUnion.size() );
-        assertEquals( first.getId(), notSecondWildcardUnion.get( 0 ).getId() );
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(notSecondWildcardUnion.get(0).getId().getType()));
+        assertEquals(first.getId().getUuid(), notSecondWildcardUnion.get(0).getId().getUuid());
 
 
         final String notSecondContains = "NOT string contains 'beer'";
@@ -1141,7 +1179,8 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 1, notSecondContainsUnion.size() );
-        assertEquals( first.getId(), notSecondContainsUnion.get( 0 ).getId() );
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(notSecondContainsUnion.get(0).getId().getType()));
+        assertEquals(first.getId().getUuid(), notSecondContainsUnion.get(0).getId().getUuid());
 
 
         final String notBothReturn = "NOT string = 'I'm a foodie'";
@@ -1151,8 +1190,10 @@ public class EntityIndexTest extends BaseIT {
 
 
         assertEquals( 2, notBothReturnResults.size() );
-        assertEquals( second.getId(), notBothReturnResults.get( 0).getId() );
-        assertEquals( first.getId(), notBothReturnResults.get( 1 ).getId() );
+        assertEquals(second.getId().getType(), CollectionUtils.stripEmptyVersion(notBothReturnResults.get(0).getId().getType()));
+        assertEquals(second.getId().getUuid(), notBothReturnResults.get(0).getId().getUuid());
+        assertEquals(first.getId().getType(), CollectionUtils.stripEmptyVersion(notBothReturnResults.get(1).getId().getType()));
+        assertEquals(first.getId().getUuid(), notBothReturnResults.get(1).getId().getUuid());
 
 
         final String notFilterBoth = "(NOT string = 'I ate a sammich') AND (NOT string = 'I drank a beer') ";
@@ -1299,7 +1340,8 @@ public class EntityIndexTest extends BaseIT {
         final CandidateResult candidate1 = candidateResults.get(0);
 
         //check the id and version
-        assertEquals( entity1.getId(), candidate1.getId() );
+        assertEquals(entity1.getId().getType(), CollectionUtils.stripEmptyVersion(candidate1.getId().getType()));
+        assertEquals(entity1.getId().getUuid(), candidate1.getId().getUuid());
         assertEquals(entity1.getVersion(), candidate1.getVersion());
 
     }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java
index ba33030..91f41b6 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java
@@ -28,6 +28,7 @@ import java.util.*;
 import org.apache.usergrid.persistence.core.CassandraFig;
 import org.apache.usergrid.persistence.index.*;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.CollectionUtils;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -101,7 +102,7 @@ public class GeoPagingTest extends BaseIT {
 
     /**
      * Test that geo-query returns co-located entities in expected order.
-     */
+     nde*/
     @Test
     public void groupQueriesWithDistanceOrderedResults() throws Exception {
 
@@ -150,7 +151,9 @@ public class GeoPagingTest extends BaseIT {
 
 
             final Entity expected = cats[consistent];
-            assertEquals(expected.getId(), candidate.getId());
+            // with collection versioning, empty versions are included
+            assertEquals(expected.getId().getType(), CollectionUtils.stripEmptyVersion(candidate.getId().getType()));
+            assertEquals(expected.getId().getUuid(), candidate.getId().getUuid());
 
 
         }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java
index d93f8a3..af94864 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexingUtilsTest.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.persistence.index.impl;
 
 import java.util.UUID;
 
+import org.apache.usergrid.persistence.model.util.CollectionUtils;
 import org.junit.Test;
 
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
@@ -87,7 +88,9 @@ public class IndexingUtilsTest {
         final CandidateResult parsedId = parseIndexDocId( output );
 
         assertEquals(version, parsedId.getVersion());
-        assertEquals(id, parsedId.getId());
+        // with collection versioning, empty versions are included when parsing doc IDs
+        assertEquals(id.getType(), CollectionUtils.stripEmptyVersion(parsedId.getId().getType()));
+        assertEquals(id.getUuid(), parsedId.getId().getUuid());
     }
 
 
@@ -119,7 +122,9 @@ public class IndexingUtilsTest {
         final CandidateResult parsedId = parseIndexDocId( output );
 
         assertEquals(version, parsedId.getVersion());
-        assertEquals(id, parsedId.getId());
+        // with collection versioning, empty versions are included when parsing doc IDs
+        assertEquals(id.getType(), CollectionUtils.stripEmptyVersion(parsedId.getId().getType()));
+        assertEquals(id.getUuid(), parsedId.getId().getUuid());
 
         final UUID appId = parseAppIdFromIndexDocId(output);
         assertEquals(appId,applicationScope.getApplication().getUuid());
@@ -154,7 +159,9 @@ public class IndexingUtilsTest {
         final CandidateResult parsedId = parseIndexDocId( output );
 
         assertEquals(version, parsedId.getVersion());
-        assertEquals(id, parsedId.getId());
+        // with collection versioning, empty versions are included when parsing doc IDs
+        assertEquals(id.getType(), CollectionUtils.stripEmptyVersion(parsedId.getId().getType()));
+        assertEquals(id.getUuid(), parsedId.getId().getUuid());
     }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/corepersistence/queue/src/test/resources/qakka.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/resources/qakka.properties b/stack/corepersistence/queue/src/test/resources/qakka.properties
index d77e7e8..f78be41 100644
--- a/stack/corepersistence/queue/src/test/resources/qakka.properties
+++ b/stack/corepersistence/queue/src/test/resources/qakka.properties
@@ -18,7 +18,7 @@
 
 # Properties for JUnit tests
 
-queue.standalone=true
+queue.standalone=false
 
 usergrid.cluster_name=Test Cluster
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
index 86b3216..f2e428d 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
@@ -31,7 +31,7 @@ import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.PathSegment;
 import javax.ws.rs.core.UriInfo;
 
-import org.apache.usergrid.persistence.entities.Application;
+import org.apache.usergrid.persistence.model.util.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.annotation.Scope;
@@ -89,7 +89,7 @@ public class CollectionResource extends ServiceResource {
             logger.trace( "CollectionResource.executeClearCollection" );
         }
 
-        if (!Application.isCustomCollectionName(itemName.getPath())) {
+        if (!CollectionUtils.isCustomCollectionOrEntityName(itemName.getPath())) {
             throw new IllegalArgumentException(
                 "Cannot clear built-in collections (" + itemName + ")."
             );
@@ -139,7 +139,7 @@ public class CollectionResource extends ServiceResource {
             logger.trace( "CollectionResource.executeGetCollectionVersion" );
         }
 
-        if (!Application.isCustomCollectionName(itemName.getPath())) {
+        if (!CollectionUtils.isCustomCollectionOrEntityName(itemName.getPath())) {
             throw new IllegalArgumentException(
                 "Built-in collections are not versioned."
             );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
index 14ed54f..306231f 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
@@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
 import com.google.cloud.storage.StorageException;
 import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.corepersistence.index.CollectionVersionUtil;
 import org.apache.usergrid.corepersistence.index.VersionedCollectionName;
 import org.apache.usergrid.management.OrganizationConfig;
 import org.apache.usergrid.management.OrganizationConfigProps;
@@ -40,6 +39,7 @@ import org.apache.usergrid.services.*;
 import org.apache.usergrid.services.assets.BinaryStoreFactory;
 import org.apache.usergrid.services.assets.data.*;
 import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
+import org.apache.usergrid.corepersistence.index.CollectionVersionUtils;
 import org.apache.usergrid.utils.JsonUtils;
 import org.glassfish.jersey.media.multipart.BodyPart;
 import org.glassfish.jersey.media.multipart.BodyPartEntity;
@@ -47,7 +47,6 @@ import org.glassfish.jersey.media.multipart.FormDataBodyPart;
 import org.glassfish.jersey.media.multipart.FormDataMultiPart;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.BeanInfoFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Scope;
 import org.springframework.stereotype.Component;
@@ -431,7 +430,7 @@ public class ServiceResource extends AbstractContextResource {
         for (int i = 0; i < r.getEntities().size(); i++) {
             Entity e = r.getEntity(i);
             String oldType = e.getType();
-            VersionedCollectionName v = CollectionVersionUtil.parseVersionedName(oldType);
+            VersionedCollectionName v = CollectionVersionUtils.parseVersionedName(oldType);
             if (v.hasVersion()) {
                 e.setType(v.getCollectionName());
                 r.setEntity(i, e);

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionClearTooSoonExceptionMapper.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionClearTooSoonExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionClearTooSoonExceptionMapper.java
new file mode 100644
index 0000000..bc49d2a
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionClearTooSoonExceptionMapper.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.exceptions;
+
+
+import org.apache.usergrid.corepersistence.asyncevents.CollectionClearTooSoonException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.Provider;
+
+import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
+
+
+@Provider
+public class CollectionClearTooSoonExceptionMapper extends AbstractExceptionMapper<CollectionClearTooSoonException> {
+
+    private static final Logger logger = LoggerFactory.getLogger(CollectionClearTooSoonExceptionMapper.class);
+
+    @Override
+    public Response toResponse( CollectionClearTooSoonException e ) {
+
+        if(logger.isTraceEnabled()) {
+            logger.trace("Tried to clear collection too soon after previous clear", e.getMessage());
+        }
+
+        return toResponse( BAD_REQUEST, e );
+    }
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionDeleteTooSoonExceptionMapper.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionDeleteTooSoonExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionDeleteTooSoonExceptionMapper.java
deleted file mode 100644
index 3b247df..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/CollectionDeleteTooSoonExceptionMapper.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.rest.exceptions;
-
-
-import org.apache.usergrid.corepersistence.asyncevents.CollectionDeleteTooSoonException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.core.Response;
-import javax.ws.rs.ext.Provider;
-
-import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
-
-
-@Provider
-public class CollectionDeleteTooSoonExceptionMapper extends AbstractExceptionMapper<CollectionDeleteTooSoonException> {
-
-    private static final Logger logger = LoggerFactory.getLogger(CollectionDeleteTooSoonExceptionMapper.class);
-
-    @Override
-    public Response toResponse( CollectionDeleteTooSoonException e ) {
-
-        if(logger.isTraceEnabled()) {
-            logger.trace("Tried to delete collection too soon after previous deletion", e.getMessage());
-        }
-
-        return toResponse( BAD_REQUEST, e );
-    }
-}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
index be60177..3752b08 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
@@ -354,6 +354,9 @@ public class IndexResource extends AbstractContextResource {
         response.setProperty( "status", status.getStatus() );
         response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
         response.setProperty( "numberQueued", status.getNumberProcessed() );
+        if (request.getUpdateTimestamp().isPresent() && request.getUpdateTimestamp().get() > 0L) {
+            response.setProperty("updatedSince", request.getUpdateTimestamp());
+        }
         response.setSuccess();
 
         return response;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionClearTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionClearTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionClearTest.java
index e40c193..b25a717 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionClearTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionClearTest.java
@@ -44,7 +44,7 @@ public class CollectionClearTest extends AbstractRestIT {
      * @throws Exception
      */
     @Test
-    public void collectionDelete() throws Exception {
+    public void collectionClear() throws Exception {
 
         String collectionName = "children";
         int numEntities = 10;
@@ -54,47 +54,98 @@ public class CollectionClearTest extends AbstractRestIT {
         String namePrefixAfterClear = "abc";
 
         // verify collection version is empty
-        ApiResponse tempResponse = this.app().collection(collectionName).collection("_version").get().getResponse();
-        LinkedHashMap dataMap = (LinkedHashMap)tempResponse.getData();
-        assertEquals("", dataMap.get("version"));
-        assertEquals(collectionName, dataMap.get("collectionName"));
+        String collectionVersion = getCollectionVersion(collectionName);
+        assertEquals("", collectionVersion);
 
-        createEntities( collectionName, namePrefix, numEntities );
+        createEntities( collectionName, namePrefix, 1, numEntities );
 
         // retrieve entities, provide 1 more than num entities
-        QueryParameters parms = new QueryParameters().setLimit( numEntities + 1 ).setQuery("order by created asc");
-        List<Entity> entities = retrieveEntities(collectionName, namePrefix, parms, numEntities, false);
+        QueryParameters parms = new QueryParameters().setLimit( numEntities + 1 );
+        List<Entity> entities = retrieveEntities(collectionName, namePrefix, parms, 1, numEntities, true);
         assertEquals(numEntities, entities.size());
 
         // clear the collection
-        Map<String, Object> payload = new HashMap<String, Object>();
+        Map<String, Object> payload = new HashMap<>();
         parms = new QueryParameters().setKeyValue("confirm_collection_name", collectionName);
-        tempResponse = this.app().collection(collectionName).collection("_clear").post(true, payload, parms);
+        ApiResponse clearResponse = this.app().collection(collectionName).collection("_clear").post(true, payload, parms);
 
         // verify collection version has changed
-        tempResponse = this.app().collection(collectionName).collection("_version").get().getResponse();
-        dataMap = (LinkedHashMap)tempResponse.getData();
-        String newVersion = (String)dataMap.get("version");
+        String newVersion = getCollectionVersion(collectionName);
         assertNotEquals("", newVersion);
-        assertEquals(collectionName, dataMap.get("collectionName"));
 
         // validate that 0 entities left
-        List<Entity> entitiesAfterClear = retrieveEntities(collectionName, namePrefix, parms, 0, true);
+        List<Entity> entitiesAfterClear = retrieveEntities(collectionName, namePrefix, parms, 1, 0, true);
         assertEquals(0, entitiesAfterClear.size());
 
         // insert more entities using same collectionName
-        createEntities( collectionName, namePrefixAfterClear, numEntitiesAfterClear );
+        createEntities( collectionName, namePrefixAfterClear, 1, numEntitiesAfterClear );
 
         // validate correct number of entities
-        parms = new QueryParameters().setLimit( numEntitiesAfterClear + 1 ).setQuery("order by created asc");
-        List<Entity> newEntities = retrieveEntities(collectionName, namePrefixAfterClear, parms, numEntitiesAfterClear, false);
+        parms = new QueryParameters().setLimit( numEntitiesAfterClear + 1 );
+        List<Entity> newEntities = retrieveEntities(collectionName, namePrefixAfterClear, parms, 1, numEntitiesAfterClear, true);
         assertEquals(numEntitiesAfterClear, newEntities.size());
 
         // verify collection version has not changed
-        tempResponse = this.app().collection(collectionName).collection("_version").get().getResponse();
-        dataMap = (LinkedHashMap)tempResponse.getData();
-        assertEquals(newVersion, dataMap.get("version"));
+        String lastVersion = getCollectionVersion(collectionName);
+        assertEquals(newVersion, lastVersion);
+    }
+
+
+    /**
+     * Tests that old collection entities are deleted.
+     * @throws Exception
+     */
+    @Test
+    public void collectionMultipleClear() throws Exception {
+        String collectionName = "dogs";
+        int numEntities = 2000;
+        String namePrefix = "dog";
+        int numDeleteCycles = 3;
+        int startingEntityNum = 1;
+
+        // should start out as unversioned
+        String currentVersion = getCollectionVersion(collectionName);
+        assertEquals("", currentVersion);
+
+        for (int cycle = 1; cycle <= numDeleteCycles; cycle++) {
+            logger.info("Creating entities {} - {} for cycle {}", startingEntityNum, lastEntityNum(startingEntityNum, numEntities), cycle);
+            createEntities( collectionName, namePrefix, startingEntityNum, numEntities );
+
+            // retrieve entities, provide 1 more than num entities
+            logger.info("Retrieving entities {} - {} for cycle {}", startingEntityNum, lastEntityNum(startingEntityNum, numEntities), cycle);
+            QueryParameters parms = new QueryParameters().setLimit( numEntities + 1 );
+            List<Entity> entities = retrieveEntities(collectionName, namePrefix, parms, startingEntityNum, numEntities, true);
+            assertEquals(numEntities, entities.size());
+
+            // clear collection
+            logger.info("Clearing collection for cycle {}", cycle);
+            String newVersion = clearCollection(collectionName);
+            logger.info("Collection version is {} for cycle {}", newVersion, cycle);
+            assertNotEquals(currentVersion, newVersion);
+
+            // validate that 0 entities left
+            List<Entity> entitiesAfterClear = retrieveEntities(collectionName, namePrefix, parms, 1, 0, true);
+            assertEquals(0, entitiesAfterClear.size());
+
+            currentVersion = newVersion;
+            startingEntityNum = startingEntityNum + numEntities;
+        }
+
+    }
+
+    private int lastEntityNum(int startingEntityNum, int numEntities) {
+        return startingEntityNum + numEntities - 1;
+    }
+
+
+    /**
+     * Get collection version
+     */
+    private String getCollectionVersion(String collectionName) {
+        ApiResponse tempResponse = this.app().collection(collectionName).collection("_version").get().getResponse();
+        LinkedHashMap dataMap = (LinkedHashMap)tempResponse.getData();
         assertEquals(collectionName, dataMap.get("collectionName"));
+        return (String)dataMap.get("version");
     }
 
 
@@ -104,10 +155,10 @@ public class CollectionClearTest extends AbstractRestIT {
      * @param collectionName
      * @param numOfEntities
      */
-    public List<Entity> createEntities(String collectionName, String namePrefix, int numOfEntities ){
+    public List<Entity> createEntities(String collectionName, String namePrefix, int firstEntity, int numOfEntities ){
         List<Entity> entities = new LinkedList<>(  );
 
-        for ( int i = 1; i <= numOfEntities; i++ ) {
+        for ( int i = firstEntity; i <= lastEntityNum(firstEntity, numOfEntities); i++ ) {
             Map<String, Object> entityPayload = new HashMap<String, Object>();
             entityPayload.put( "name", namePrefix + String.valueOf( i ) );
             entityPayload.put( "num", i );
@@ -117,12 +168,8 @@ public class CollectionClearTest extends AbstractRestIT {
             entities.add( entity );
 
             this.app().collection( collectionName ).post( entity );
-
-            if ( i % 100 == 0){
-                logger.info("created {} entities", i);
-            }
         }
-        logger.info("created {} total entities", numOfEntities);
+        logger.info("created {} entities", numOfEntities);
 
         this.waitForQueueDrainAndRefreshIndex();
 
@@ -135,15 +182,15 @@ public class CollectionClearTest extends AbstractRestIT {
      * @param parms
      * @param numOfEntities
      */
-    public List<Entity> retrieveEntities(String collectionName, String namePrefix, QueryParameters parms, int numOfEntities, boolean reverseOrder){
+    public List<Entity> retrieveEntities(String collectionName, String namePrefix, QueryParameters parms, int firstEntity, int numOfEntities, boolean reverseOrder){
         List<Entity> entities = new LinkedList<>(  );
         Collection testCollection = this.app().collection( collectionName ).get(parms, true);
 
         int entityNum;
         if (reverseOrder) {
-            entityNum = numOfEntities;
+            entityNum = lastEntityNum(firstEntity, numOfEntities);
         } else {
-            entityNum = 1;
+            entityNum = firstEntity;
         }
         while (testCollection.getCursor() != null) {
             while (testCollection.hasNext()) {
@@ -176,4 +223,16 @@ public class CollectionClearTest extends AbstractRestIT {
         return entities;
     }
 
+    private String clearCollection(String collectionName) {
+        // clear the collection
+        Map<String, Object> payload = new HashMap<>();
+        QueryParameters parms = new QueryParameters().setKeyValue("confirm_collection_name", collectionName);
+        ApiResponse clearResponse = this.app().collection(collectionName).collection("_clear").post(true, payload, parms);
+
+        // verify collection version has changed
+        String newVersion = getCollectionVersion(collectionName);
+
+        return newVersion;
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
index dee78f9..10ac63a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
@@ -17,6 +17,7 @@
 package org.apache.usergrid.services;
 
 
+import org.apache.usergrid.corepersistence.index.CollectionVersionUtils;
 import org.apache.usergrid.persistence.*;
 import org.apache.usergrid.persistence.Query.Level;
 import org.apache.usergrid.persistence.index.query.Identifier;
@@ -32,13 +33,16 @@ import rx.Observable;
 import rx.schedulers.Schedulers;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 
+import static org.apache.usergrid.services.ServiceManager.MAP_VERSIONED_COLLECTION_NAME_KEY;
 import static org.apache.usergrid.services.ServiceParameter.filter;
 import static org.apache.usergrid.services.ServiceParameter.firstParameterIsName;
 import static org.apache.usergrid.utils.ClassUtils.cast;
 import static org.apache.usergrid.utils.InflectionUtils.pluralize;
+import static org.apache.usergrid.utils.InflectionUtils.singularize;
 import static org.apache.usergrid.utils.ListUtils.dequeue;
 import static org.apache.usergrid.utils.ListUtils.initCopy;
 
@@ -90,6 +94,9 @@ public class AbstractConnectionsService extends AbstractService {
             cType = dequeue( parameters ).getName();
         }
         if ( cType != null ) {
+            // this is not a versionable collection. If there is a version here, it is because the connection name matches
+            // a versioned collection. Remove the version.
+            cType = CollectionVersionUtils.getBaseCollectionName(cType);
             collectionName = cType;
         }
 
@@ -121,6 +128,9 @@ public class AbstractConnectionsService extends AbstractService {
             }
             else {
                 eType = Schema.normalizeEntityType( s );
+                Map<String,String> collectionInfo = sm.getVersionedCollectionInfo(s, eType);
+                eType = collectionInfo.get(MAP_VERSIONED_COLLECTION_NAME_KEY);
+                //logger.info("connection service collection eType:{}", eType);
                 first_parameter = dequeue( parameters );
                 if ( first_parameter instanceof QueryParameter ) {
                     query = first_parameter.getQuery();

http://git-wip-us.apache.org/repos/asf/usergrid/blob/24e443b2/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
index 711a86c..1f5aeb4 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
@@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit;
 import org.apache.usergrid.corepersistence.index.CollectionScopeImpl;
 import org.apache.usergrid.corepersistence.index.CollectionVersionManager;
 import org.apache.usergrid.corepersistence.index.CollectionVersionManagerFactory;
-import org.apache.usergrid.corepersistence.index.CollectionVersionUtil;
+import org.apache.usergrid.corepersistence.index.CollectionVersionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.ApplicationContext;
@@ -49,6 +49,7 @@ import com.google.common.cache.LoadingCache;
 
 import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
 import static org.apache.usergrid.utils.InflectionUtils.pluralize;
+import static org.apache.usergrid.utils.InflectionUtils.singularize;
 
 
 public class ServiceManager {
@@ -225,6 +226,57 @@ public class ServiceManager {
     }
 
 
+    public static String MAP_VERSIONED_COLLECTION_NAME_KEY = "versionedName";
+    public static String MAP_UNVERSIONED_COLLECTION_NAME_KEY = "unversionedName";
+    public static String MAP_VERSIONED_ITEM_TYPE_KEY = "versionedType";
+    public static String MAP_UNVERSIONED_ITEM_TYPE_KEY = "unversionedType";
+    public static String MAP_COLLECTION_VERSION_KEY = "version";
+
+    /**
+     * Returns versioned collection info in Map (
+     * @param collectionName
+     * @param itemType
+     * @return
+     */
+    public Map<String,String> getVersionedCollectionInfo(String collectionName, String itemType) {
+        Map<String,String> collectionInfo = new HashMap<>();
+        String versionedCollectionName = collectionName;
+        String unversionedCollectionName = CollectionVersionUtils.getBaseCollectionName(collectionName);
+        String versionedItemType = itemType;
+        // this works for item type too
+        String unversionedItemType = CollectionVersionUtils.getBaseCollectionName(itemType);
+        String collectionVersion = "";
+        if (collectionName.equals(unversionedCollectionName)) {
+            // no version passed in
+            CollectionVersionManager collectionVersionManager = cvmf.getInstance(new CollectionScopeImpl(applicationId, unversionedCollectionName));
+            // always bypass collection version cache for now
+            collectionVersion = collectionVersionManager.getCollectionVersion(true);
+
+            if (collectionVersion != "") {
+                if (logger.isTraceEnabled()) {
+                    logger.trace("getVersionedCollectionName: currentCollectionVersion={}", collectionVersion);
+                }
+                versionedCollectionName = CollectionVersionUtils.buildVersionedNameString(unversionedCollectionName, collectionVersion, false);
+                versionedItemType = CollectionVersionUtils.buildVersionedNameString(itemType, collectionVersion, false);
+                if (logger.isTraceEnabled()) {
+                    logger.trace("getVersionedCollectionName() - using versioned collection name: collectionName={} versionedCollectionName={} versionedItemType={}",
+                        unversionedCollectionName, versionedCollectionName, versionedItemType);
+                }
+            }
+        } else {
+            // version was passed in
+            collectionVersion = CollectionVersionUtils.getCollectionVersion(versionedCollectionName);
+        }
+        collectionInfo.put(MAP_VERSIONED_COLLECTION_NAME_KEY, versionedCollectionName);
+        collectionInfo.put(MAP_UNVERSIONED_COLLECTION_NAME_KEY, unversionedCollectionName);
+        collectionInfo.put(MAP_VERSIONED_ITEM_TYPE_KEY, versionedItemType);
+        collectionInfo.put(MAP_UNVERSIONED_ITEM_TYPE_KEY, unversionedItemType);
+        collectionInfo.put(MAP_COLLECTION_VERSION_KEY, collectionVersion);
+
+        return collectionInfo;
+    }
+
+
     public Service getService( String serviceType ) {
         return getService( serviceType, true );
     }
@@ -246,32 +298,15 @@ public class ServiceManager {
             return null;
         }
 
-        // use versionedCollectionName if appropriate
-        String versionedCollectionName = info.getCollectionName();
-        String unversionedCollectionName = CollectionVersionUtil.getBaseCollectionName(versionedCollectionName);
+        Map<String,String> collectionInfo = getVersionedCollectionInfo(info.getCollectionName(), info.getItemType());
+        String versionedCollectionName = collectionInfo.get(MAP_VERSIONED_COLLECTION_NAME_KEY);
+        String versionedItemType = collectionInfo.get(MAP_VERSIONED_ITEM_TYPE_KEY);
         if (logger.isTraceEnabled()) {
             logger.trace("getService: serviceType={} incoming collectionName={}", serviceType, versionedCollectionName);
         }
 
-        // if versioned collection name was passed in, use it, because it may be for an old version
-        if (versionedCollectionName.equals(unversionedCollectionName)) {
-            // no version passed in
-            CollectionVersionManager collectionVersionManager = cvmf.getInstance(new CollectionScopeImpl(applicationId, unversionedCollectionName));
-            // always bypass collection version cache for now
-            String currentCollectionVersion = collectionVersionManager.getCollectionVersion(true);
-
-            if (currentCollectionVersion != "") {
-                if (logger.isTraceEnabled()) {
-                    logger.trace("getService: currentCollectionVersion={}", currentCollectionVersion);
-                }
-                versionedCollectionName = CollectionVersionUtil.buildVersionedNameString(unversionedCollectionName, currentCollectionVersion, false);
-                String versionedItemType = CollectionVersionUtil.buildVersionedNameString(info.getItemType(), currentCollectionVersion, false);
-                if (logger.isTraceEnabled()) {
-                    logger.trace("getService() - using versioned collection name: collectionName={} versionedCollectionName={} versionedItemType={}",
-                        unversionedCollectionName, versionedCollectionName, versionedItemType);
-                }
-                info = ServiceInfo.getVersionedServiceInfo(info, versionedCollectionName, versionedItemType);
-            }
+        if (!versionedCollectionName.equals(info.getCollectionName())) {
+            info = ServiceInfo.getVersionedServiceInfo(info, versionedCollectionName, versionedItemType);
         }
 
         Service service = getServiceInstance( info );


Mime
View raw message