usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From toddn...@apache.org
Subject incubator-usergrid git commit: Revert of geo per shard. Geo has a hashing bug on write, and therefore does not correctly allocate the same shard as the rest of the entity on write. Aggregate + filter is the only mechanism to return correct results witho
Date Wed, 01 Jul 2015 17:02:52 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/USERGRID-752 9971068a0 -> 595aa71d6


Revert of geo per shard. Geo has a hashing bug on write, and therefore does not correctly
allocate the same shard as the rest of the entity on write.  Aggregate + filter is the only
mechanism to return correct results without triggering a complete re-index.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/595aa71d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/595aa71d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/595aa71d

Branch: refs/heads/USERGRID-752
Commit: 595aa71d6512ed5f43f9ff252d28824094cf7223
Parents: 9971068
Author: Todd Nine <tnine@apigee.com>
Authored: Wed Jul 1 11:02:48 2015 -0600
Committer: Todd Nine <tnine@apigee.com>
Committed: Wed Jul 1 11:02:48 2015 -0600

----------------------------------------------------------------------
 .../persistence/geo/CollectionGeoSearch.java    |   4 +-
 .../persistence/geo/ConnectionGeoSearch.java    |   4 +-
 .../persistence/geo/GeoIndexSearcher.java       |  13 +-
 .../query/ir/result/CollectionShardFilter.java  |  53 ++++++
 .../ir/result/SearchCollectionVisitor.java      |  12 +-
 .../ir/result/SearchConnectionVisitor.java      |   9 +-
 .../org/apache/usergrid/persistence/GeoIT.java  | 161 +++++++++----------
 7 files changed, 154 insertions(+), 102 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
index d8635d9..c823e20 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/CollectionGeoSearch.java
@@ -44,9 +44,9 @@ public class CollectionGeoSearch extends GeoIndexSearcher {
     private final EntityRef headEntity;
 
 
-    public CollectionGeoSearch( EntityManager entityManager, final String shard, CassandraService
cass,
+    public CollectionGeoSearch( EntityManager entityManager, IndexBucketLocator locator,
CassandraService cass,
                                 EntityRef headEntity, String collectionName ) {
-        super( entityManager, shard, cass );
+        super( entityManager, locator, cass );
         this.collectionName = collectionName;
         this.headEntity = headEntity;
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
index 37b96dd..a1ad71e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/ConnectionGeoSearch.java
@@ -43,9 +43,9 @@ public class ConnectionGeoSearch extends GeoIndexSearcher {
     private final UUID connectionId;
 
 
-    public ConnectionGeoSearch( EntityManager entityManager,  final String shard, CassandraService
cass,
+    public ConnectionGeoSearch( EntityManager entityManager, IndexBucketLocator locator,
CassandraService cass,
                                 UUID connectionId ) {
-        super( entityManager, shard, cass );
+        super( entityManager, locator, cass );
 
         this.connectionId = connectionId;
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
index e61aa64..eb20ffa 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/geo/GeoIndexSearcher.java
@@ -65,12 +65,12 @@ public abstract class GeoIndexSearcher {
     private static final int MAX_FETCH_SIZE = 1000;
 
     protected final EntityManager em;
-    protected final String shard;
+    protected final IndexBucketLocator locator;
     protected final CassandraService cass;
 
-    public GeoIndexSearcher(final  EntityManager entityManager, final String shard, final
CassandraService cass ) {
+    public GeoIndexSearcher( EntityManager entityManager, IndexBucketLocator locator, CassandraService
cass ) {
         this.em = entityManager;
-        this.shard = shard;
+        this.locator = locator;
         this.cass = cass;
     }
 
@@ -325,7 +325,12 @@ public abstract class GeoIndexSearcher {
         for ( String geoCell : curGeocellsUnique ) {
 
             // add buckets for each geoCell
-            keys.add( key( key, DICTIONARY_GEOCELL, geoCell, shard ) );
+
+            //TODO, use merge logic here
+
+            for ( String indexBucket : locator.getBuckets( ) ) {
+                keys.add( key( key, DICTIONARY_GEOCELL, geoCell, indexBucket ) );
+            }
         }
 
         DynamicComposite start = null;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionShardFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionShardFilter.java
b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionShardFilter.java
new file mode 100644
index 0000000..cc8b7c7
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/CollectionShardFilter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.persistence.query.ir.result;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
+
+
+/**
+ * Class that performs validation on an entity to ensure it's in the shard we expect
+ */
+public final class CollectionShardFilter implements ShardFilter {
+    private final IndexBucketLocator indexBucketLocator;
+    private final String expectedBucket;
+
+
+    public CollectionShardFilter( final IndexBucketLocator indexBucketLocator, final String
expectedBucket ) {
+        this.indexBucketLocator = indexBucketLocator;
+        this.expectedBucket = expectedBucket;
+    }
+
+
+
+
+
+    public boolean isInShard( final ScanColumn scanColumn ) {
+
+        final UUID entityId = scanColumn.getUUID();
+
+        //not for our current processing shard, discard
+        final String shard = indexBucketLocator.getBucket( entityId );
+
+        return expectedBucket.equals( shard );
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
index 73eb148..32c3a6e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchCollectionVisitor.java
@@ -128,16 +128,16 @@ public class SearchCollectionVisitor extends SearchVisitor {
         final int size = queryProcessor.getPageSizeHint( node );
 
         GeoIterator itr = new GeoIterator(
-                new CollectionGeoSearch( em, bucket, cassandraService, headEntity, collection.getName()
),
+                new CollectionGeoSearch( em, indexBucketLocator, cassandraService, headEntity,
collection.getName() ),
                 size, slice, node.getPropertyName(), new Point( node.getLattitude(), node.getLongitude()
),
                 node.getDistance() );
 
-        this.results.push( itr );
+//        this.results.push( itr );
 
-//        final CollectionShardFilter
-//                validator = new CollectionShardFilter(indexBucketLocator, bucket );
-//
-//        this.results.push(  new ShardFilterIterator( validator, itr, size));
+        final CollectionShardFilter
+                validator = new CollectionShardFilter(indexBucketLocator, bucket );
+
+        this.results.push(  new ShardFilterIterator( validator, itr, size));
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
index 404420a..5505914 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/SearchConnectionVisitor.java
@@ -115,17 +115,16 @@ public class SearchConnectionVisitor extends SearchVisitor {
 
         //TODO, make search take a shard
         GeoIterator itr =
-                new GeoIterator( new ConnectionGeoSearch( em, bucket, cassandraService, connection.getIndexId()
),
+                new GeoIterator( new ConnectionGeoSearch( em, indexBucketLocator, cassandraService,
connection.getIndexId() ),
                         size, slice, node.getPropertyName(),
                         new Point( node.getLattitude(), node.getLongitude() ), node.getDistance()
);
 
 
-        this.results.push( itr );
-//        final CollectionShardFilter
-//                validator = new CollectionShardFilter(indexBucketLocator, bucket );
+        final ConnectionShardFilter
+                validator = new ConnectionShardFilter(indexBucketLocator, bucket, connection
);
 
 
-//        this.results.push( new ShardFilterIterator( validator, itr, size ) );
+        this.results.push( new ShardFilterIterator( validator, itr, size ) );
 
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/595aa71d/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
index 8e6fb80..8c9e9b7 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
@@ -76,10 +76,8 @@ public class GeoIT extends AbstractCoreIT {
 
         Point center = new Point( 37.774277, -122.404744 );
 
-        final String shard = setup.getIbl().getBucket( user.getUuid() );
-
         CollectionGeoSearch connSearch =
-                new CollectionGeoSearch( em, shard, setup.getCassSvc(), em.getApplicationRef(),
"users" );
+                new CollectionGeoSearch( em, setup.getIbl(), setup.getCassSvc(), em.getApplicationRef(),
"users" );
 
 
         List<EntityLocationRef> listResults =
@@ -421,86 +419,83 @@ public class GeoIT extends AbstractCoreIT {
     }
 
 
-//    @Test
-//    public void testDenseSearch() throws Exception {
-//
-//        UUID applicationId = setup.createApplication( "testOrganization", "testDenseSearch"
);
-//        assertNotNull( applicationId );
-//
-//        EntityManager em = setup.getEmf().getEntityManager( applicationId );
-//        assertNotNull( em );
-//
-//        // save objects in a diagonal line from -90 -180 to 90 180
-//
-//        int numEntities = 500;
-//
-//        float minLattitude = 48.32455f;
-//        float maxLattitude = 48.46481f;
-//        float minLongitude = 9.89561f;
-//        float maxLongitude = 10.0471f;
-//
-//        float lattitudeDelta = ( maxLattitude - minLattitude ) / numEntities;
-//
-//        float longitudeDelta = ( maxLongitude - minLongitude ) / numEntities;
-//
-//        for ( int i = 0; i < numEntities; i++ ) {
-//            float lattitude = minLattitude + lattitudeDelta * i;
-//            float longitude = minLongitude + longitudeDelta * i;
-//
-//            Map<String, Float> location = MapUtils.hashMap( "latitude", lattitude
).map( "longitude", longitude );
-//
-//            Map<String, Object> data = new HashMap<String, Object>( 2 );
-//            data.put( "name", String.valueOf( i ) );
-//            data.put( "location", location );
-//
-//            em.create( "store", data );
-//        }
-//
-//        //do a direct geo iterator test.  We need to make sure that we short circuit on
the correct tile.
-//
-//        float lattitude = 48.38626f;
-//        float longtitude = 9.94175f;
-//        int distance = 1000;
-//        int limit = 8;
-//
-//
-//        QuerySlice slice = new QuerySlice( "location", 0 );
-//
-//        //TODO test this
-//        final String shard = setup.getIbl().getBucket( user.getUuid() );
-//
-//        GeoIterator itr = new GeoIterator(
-//                new CollectionGeoSearch( em, setup.getIbl(), setup.getCassSvc(), em.getApplicationRef(),
"stores" ),
-//                limit, slice, "location", new Point( lattitude, longtitude ), distance
);
-//
-//
-//        // check we got back all 500 entities
-//        assertFalse( itr.hasNext() );
-//
-//        List<String> cells = itr.getLastCellsSearched();
-//
-//        assertEquals( 1, cells.size() );
-//
-//        assertEquals( 4, cells.get( 0 ).length() );
-//
-//
-//        long startTime = System.currentTimeMillis();
-//
-//        //now test at the EM level, there should be 0 results.
-//        Query query = new Query();
-//
-//        query.addFilter( "location within 1000 of 48.38626, 9.94175" );
-//        query.setLimit( 8 );
-//
-//
-//        Results results = em.searchCollection( em.getApplicationRef(), "stores", query
);
-//
-//        assertEquals( 0, results.size() );
-//
-//        long endTime = System.currentTimeMillis();
-//
-//        LOG.info( "Runtime took {} milliseconds to search", endTime - startTime );
-//    }
+    @Test
+    public void testDenseSearch() throws Exception {
+
+        UUID applicationId = setup.createApplication( "testOrganization", "testDenseSearch"
);
+        assertNotNull( applicationId );
+
+        EntityManager em = setup.getEmf().getEntityManager( applicationId );
+        assertNotNull( em );
+
+        // save objects in a diagonal line from -90 -180 to 90 180
+
+        int numEntities = 500;
+
+        float minLattitude = 48.32455f;
+        float maxLattitude = 48.46481f;
+        float minLongitude = 9.89561f;
+        float maxLongitude = 10.0471f;
+
+        float lattitudeDelta = ( maxLattitude - minLattitude ) / numEntities;
+
+        float longitudeDelta = ( maxLongitude - minLongitude ) / numEntities;
+
+        for ( int i = 0; i < numEntities; i++ ) {
+            float lattitude = minLattitude + lattitudeDelta * i;
+            float longitude = minLongitude + longitudeDelta * i;
+
+            Map<String, Float> location = MapUtils.hashMap( "latitude", lattitude ).map(
"longitude", longitude );
+
+            Map<String, Object> data = new HashMap<String, Object>( 2 );
+            data.put( "name", String.valueOf( i ) );
+            data.put( "location", location );
+
+            em.create( "store", data );
+        }
+
+        //do a direct geo iterator test.  We need to make sure that we short circuit on the
correct tile.
+
+        float lattitude = 48.38626f;
+        float longtitude = 9.94175f;
+        int distance = 1000;
+        int limit = 8;
+
+
+        QuerySlice slice = new QuerySlice( "location", 0 );
+
+        GeoIterator itr = new GeoIterator(
+                new CollectionGeoSearch( em, setup.getIbl(), setup.getCassSvc(), em.getApplicationRef(),
"stores" ),
+                limit, slice, "location", new Point( lattitude, longtitude ), distance );
+
+
+        // check we got back all 500 entities
+        assertFalse( itr.hasNext() );
+
+        List<String> cells = itr.getLastCellsSearched();
+
+        assertEquals( 1, cells.size() );
+
+        assertEquals( 4, cells.get( 0 ).length() );
+
+
+        long startTime = System.currentTimeMillis();
+
+        //now test at the EM level, there should be 0 results.
+        Query query = new Query();
+
+        query.addFilter( "location within 1000 of 48.38626, 9.94175" );
+        query.setLimit( 8 );
+
+
+        Results results = em.searchCollection( em.getApplicationRef(), "stores", query );
+
+        assertEquals( 0, results.size() );
+
+        long endTime = System.currentTimeMillis();
+
+        LOG.info( "Runtime took {} milliseconds to search", endTime - startTime );
+    }
 
 
     public Map<String, Object> getLocation( double latitude, double longitude ) throws
Exception {


Mime
View raw message