usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sfeld...@apache.org
Subject [1/3] incubator-usergrid git commit: Fixes issue with invalid UUID test
Date Mon, 01 Jun 2015 21:56:23 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/two-dot-o-dev 45ce5d288 -> 42f45efe8


Fixes issue with invalid UUID test

Fixes GeoIT tests


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/29e885ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/29e885ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/29e885ff

Branch: refs/heads/two-dot-o-dev
Commit: 29e885ff2f002ecc8809ba51573bc33022ae6ff6
Parents: bfe1337
Author: Todd Nine <tnine@apigee.com>
Authored: Mon Jun 1 13:37:15 2015 -0600
Committer: Todd Nine <tnine@apigee.com>
Committed: Mon Jun 1 13:37:15 2015 -0600

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        |  2 +-
 .../corepersistence/util/CpNamingUtils.java     |  5 +-
 .../org/apache/usergrid/utils/UUIDUtils.java    | 32 ++---------
 .../org/apache/usergrid/persistence/GeoIT.java  |  5 +-
 .../apache/usergrid/utils/UUIDUtilsTest.java    | 26 ++++-----
 stack/core/src/test/resources/log4j.properties  |  1 +
 .../persistence/model/util/UUIDGenerator.java   | 58 ++------------------
 .../impl/SearchRequestBuilderStrategy.java      | 50 +++++++----------
 8 files changed, 48 insertions(+), 131 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 14d1bed..50f04ed 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2500,7 +2500,7 @@ public class CpEntityManager implements EntityManager {
 
         long timestamp = UUIDUtils.getUUIDLong( timestampUuid );
 
-        UUID itemId = UUIDUtils.newTimeUUID();
+        UUID itemId = UUIDGenerator.newTimeUUID();
 
         if ( is_application ) {
             itemId = applicationId;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index 31dabea..471eda6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -39,6 +39,7 @@ import org.apache.usergrid.persistence.map.MapScope;
 import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 import org.apache.usergrid.utils.UUIDUtils;
 
 
@@ -141,7 +142,7 @@ public class CpNamingUtils {
         //if they don't use a time based uuid (such as in devices) we need to create a timestamp
from "now" since
         // this is when the entity
         //will be added to the collection
-        final UUID timeStampUuid = UUIDUtils.isTimeBased( entityIdUUID ) ? entityIdUUID :
UUIDUtils.newTimeUUID();
+        final UUID timeStampUuid = UUIDUtils.isTimeBased( entityIdUUID ) ? entityIdUUID :
UUIDGenerator.newTimeUUID();
 
         long uuidTimestamp = UUIDUtils.getUUIDLong( timeStampUuid );
 
@@ -167,7 +168,7 @@ public class CpNamingUtils {
         final String edgeType = getEdgeTypeFromConnectionType( connectionType );
 
         // create graph edge connection from head entity to member entity
-        return new SimpleEdge( sourceEntityId, edgeType, targetEntityId, System.currentTimeMillis()
);
+        return new SimpleEdge( sourceEntityId, edgeType, targetEntityId, UUIDGenerator.newTimeUUID().timestamp()
);
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java b/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java
index ecb1f61..adde4b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/utils/UUIDUtils.java
@@ -37,6 +37,8 @@ import static com.fasterxml.uuid.impl.UUIDUtil.BYTE_OFFSET_CLOCK_SEQUENCE;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
 import static org.apache.commons.codec.binary.Base64.decodeBase64;
 import static org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString;
 
@@ -82,27 +84,8 @@ public class UUIDUtils {
      * 'unique.'
      */
     public static java.util.UUID newTimeUUID() {
-        // get & inc counter, but roll on 1k (because we divide by 10 on retrieval)
-        // if count + currentMicro > 1k, block and roll
-        tsLock.lock();
-        long ts = System.currentTimeMillis();
-        if ( ts > timestampMillisNow ) {
-            timestampMillisNow = ts;
-            currentMicrosPoint.set( 0 );
-        }
-        int pointer = currentMicrosPoint.getAndIncrement();
-        try {
-            if ( pointer > 990 ) {
-                TimeUnit.MILLISECONDS.sleep( 1L );
-            }
-        }
-        catch ( Exception ex ) {
-            ex.printStackTrace();
-        }
-        finally {
-            tsLock.unlock();
-        }
-        return newTimeUUID( ts, MICROS[pointer] );
+        //replaced with our new impl.  The old impl does not properly generate sequential
times when under heavy load in the same millisecond.  Causes edge ordering issues. See USERGRID-628
+        return UUIDGenerator.newTimeUUID();
     }
 
 
@@ -210,13 +193,6 @@ public class UUIDUtils {
     }
 
 
-    public static UUID minTimeUUID( long ts ) {
-        byte[] uuidBytes = new byte[16];
-        setTimestamp( ts, uuidBytes, 0, 0 );
-
-        return uuid( uuidBytes );
-    }
-
 
     public static UUID maxTimeUUID( long ts ) {
         byte[] uuidBytes = new byte[16];

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
index c837ab7..542a1c5 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
@@ -441,6 +441,7 @@ public class GeoIT extends AbstractCoreIT {
             setPos(data, 0, 0);
 
             em.create("store", data);
+//            Thread.sleep(1);
         }
 
         app.refreshIndex();
@@ -459,7 +460,9 @@ public class GeoIT extends AbstractCoreIT {
 
             for (Entity entity : results.getEntities()) {
                 //TODO:can we assert order
-                assertEquals(String.valueOf(count), entity.getName());
+                final int expected = numEntities - count - 1;
+
+                assertEquals(String.valueOf(expected), entity.getName());
                 count++;
             }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/core/src/test/java/org/apache/usergrid/utils/UUIDUtilsTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/utils/UUIDUtilsTest.java b/stack/core/src/test/java/org/apache/usergrid/utils/UUIDUtilsTest.java
index e59dac8..0493d31 100644
--- a/stack/core/src/test/java/org/apache/usergrid/utils/UUIDUtilsTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/utils/UUIDUtilsTest.java
@@ -22,7 +22,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.UUID;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
@@ -166,10 +165,10 @@ public class UUIDUtilsTest {
 
     /** Populate timestamp set for the methods testing uuid contention */
     @SuppressWarnings("unchecked")
-    private static Set buildTsMicros( int count ) {
-        HashSet created = new HashSet( count );
+    private static Set<UUID> buildTsMicros( int count ) {
+        HashSet<UUID> created = new HashSet<>( count );
         for ( int x = 0; x < count; x++ ) {
-            created.add( UUIDUtils.getTimestampInMicros( UUIDUtils.newTimeUUID() ) );
+            created.add(  UUIDUtils.newTimeUUID() );
         }
         return created;
     }
@@ -207,20 +206,17 @@ public class UUIDUtilsTest {
         List<Future> jobs = new ArrayList<Future>( 10 );
 
         for ( int x = 0; x < 10; x++ ) {
-            jobs.add( exec.submit( new Callable<Object>() {
-                @Override
-                public Object call() throws Exception {
-                    logger.info("call invoked");
+            jobs.add( exec.submit( () -> {
+                logger.info("call invoked");
 
-                    int count = 1000 * 100;
-                    Set created = buildTsMicros( count );
+                int count = 1000 * 100;
+                Set created = buildTsMicros( count );
 
-                    assertEquals( count, created.size() );
-                    assertTrue( created.size() > 0 );
+                assertEquals( count, created.size() );
+                assertTrue( created.size() > 0 );
 
-                    logger.info("run complete");
-                    return null;
-                }
+                logger.info("run complete");
+                return null;
             } ) );
         }
         return jobs;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/log4j.properties b/stack/core/src/test/resources/log4j.properties
index 34bd708..dfcb260 100644
--- a/stack/core/src/test/resources/log4j.properties
+++ b/stack/core/src/test/resources/log4j.properties
@@ -66,3 +66,4 @@ log4j.logger.org.apache.usergrid.persistence.index.impl.EsApplicationEntityIndex
 #log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG, stdout
 log4j.logger.org.apache.usergrid.corepersistence.index.IndexServiceImpl=DEBUG
 
+log4j.logger.org.apache.usergrid.utils.UUIDUtilsTest=DEBUG

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/UUIDGenerator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/UUIDGenerator.java
b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/UUIDGenerator.java
index f2c4fa8..176c97b 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/UUIDGenerator.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/util/UUIDGenerator.java
@@ -4,86 +4,36 @@ package org.apache.usergrid.persistence.model.util;
 import java.io.IOException;
 import java.util.Random;
 import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import com.fasterxml.uuid.EthernetAddress;
-import com.fasterxml.uuid.TimestampSynchronizer;
 import com.fasterxml.uuid.UUIDTimer;
 import com.fasterxml.uuid.impl.TimeBasedGenerator;
 
 
 /**
  * TODO replace this with the Astyanax generator libs
- * @author: tnine
- *
  */
 public class UUIDGenerator {
 
 
-    private static final TimestampSynchronizer synchronize = new TimestampSynchronizer()
{
-
-        /**
-         * Pointer to the last value we returned
-         */
-        private long last = 0;
-
-        /**
-         * The number of ticks that can be used in the millisecond.  In a time UUID a tick
is divided into 1/10000 of
-         * a millisecond
-         *
-         */
-        private AtomicInteger ticks = new AtomicInteger();
-
-
-        @Override
-        protected long initialize() throws IOException {
-
-            last = System.currentTimeMillis();
-            return last;
-        }
-
-
-        @Override
-        protected void deactivate() throws IOException {
-            //no op
-        }
-
-
-        @Override
-        protected long update( long now ) throws IOException {
-            /**
-             * Our timestamp is greater just use that and reset last
-             */
-            if ( now > last ) {
-                last = now;
-                ticks.set( 0 );
-                return last;
-            }
-
-            //we have the same value (since now should always be increasing) increment a
tick
-            return last + ticks.incrementAndGet();
-        }
-    };
-
-
     private static final Random random = new Random();
     private static final UUIDTimer timer;
 
-
     /**
      * Lame, but required
      */
     static {
         try {
-            timer = new UUIDTimer( random, synchronize );
+            timer = new UUIDTimer( random, null );
         }
         catch ( IOException e ) {
-            throw new RuntimeException( "Couldn't intialize timer", e );
+            throw new RuntimeException( "Couldn't initialize timer", e );
         }
     }
 
 
-    private static final TimeBasedGenerator generator = new TimeBasedGenerator( EthernetAddress.fromInterface(),
timer );
+    private static final TimeBasedGenerator generator =
+        new TimeBasedGenerator( EthernetAddress.fromInterface(), timer );
 
 
     /** Create a new time uuid */

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/29e885ff/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
index 33a9adf..2ac6bcb 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -20,12 +20,8 @@
 package org.apache.usergrid.persistence.index.impl;
 
 
-import java.util.Set;
-
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.common.geo.GeoDistance;
-import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.index.query.BoolFilterBuilder;
 import org.elasticsearch.index.query.FilterBuilder;
 import org.elasticsearch.index.query.FilterBuilders;
@@ -82,13 +78,14 @@ public class SearchRequestBuilderStrategy {
      * Get the search request builder
      */
     public SearchRequestBuilder getBuilder( final SearchEdge searchEdge, final SearchTypes
searchTypes,
-                                            final ParsedQuery query, final int limit, final
int from) {
+                                            final ParsedQuery query, final int limit, final
int from ) {
 
         Preconditions
-                .checkArgument( limit <= EntityIndex.MAX_LIMIT, "limit is greater than
max " + EntityIndex.MAX_LIMIT );
+            .checkArgument( limit <= EntityIndex.MAX_LIMIT, "limit is greater than max
" + EntityIndex.MAX_LIMIT );
 
         SearchRequestBuilder srb =
-                esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes( IndexingUtils.ES_ENTITY_TYPE
).setSearchType(SearchType.QUERY_THEN_FETCH);
+            esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes( IndexingUtils.ES_ENTITY_TYPE
)
+                      .setSearchType( SearchType.QUERY_THEN_FETCH );
 
 
         final QueryVisitor visitor = visitParsedQuery( query );
@@ -126,28 +123,22 @@ public class SearchRequestBuilderStrategy {
      * Apply our default sort predicate logic
      */
     private void applyDefaultSortPredicates( final SearchRequestBuilder srb, final GeoSortFields
geoFields ) {
-
-
-        if ( geoFields.isEmpty() ) {
-
-            //sort by the edge timestamp
-            srb.addSort( SortBuilders.fieldSort( IndexingUtils.EDGE_TIMESTAMP_FIELDNAME ).order(
SortOrder.DESC ) );
-
-            //sort by the entity id if our times are equal
-            srb.addSort( SortBuilders.fieldSort( IndexingUtils.ENTITY_ID_FIELDNAME ).order(
SortOrder.ASC ) );
-
-            return;
-        }
-
         //we have geo fields, sort through them in visit order
-
-
         for ( String geoField : geoFields.fields() ) {
 
             final GeoDistanceSortBuilder geoSort = geoFields.applyOrder( geoField, SortOrder.ASC
);
 
             srb.addSort( geoSort );
         }
+
+        //now sort by edge timestamp, then entity id
+        //sort by the edge timestamp
+        srb.addSort( SortBuilders.fieldSort( IndexingUtils.EDGE_TIMESTAMP_FIELDNAME ).order(
SortOrder.DESC ) );
+
+        //sort by the entity id if our times are equal
+        srb.addSort( SortBuilders.fieldSort( IndexingUtils.ENTITY_ID_FIELDNAME ).order( SortOrder.ASC
) );
+
+        return;
     }
 
 
@@ -170,9 +161,10 @@ public class SearchRequestBuilderStrategy {
             final String propertyName = sp.getPropertyName();
 
 
-            //if the user specified a geo field in their sort, then honor their sort order
and use the point they specified
+            //if the user specified a geo field in their sort, then honor their sort order
and use the point they
+            // specified
             if ( geoFields.contains( propertyName ) ) {
-                final GeoDistanceSortBuilder geoSort = geoFields.applyOrder(propertyName,
 SortOrder.ASC);
+                final GeoDistanceSortBuilder geoSort = geoFields.applyOrder( propertyName,
SortOrder.ASC );
                 srb.addSort( geoSort );
             }
 
@@ -202,8 +194,8 @@ public class SearchRequestBuilderStrategy {
      * Create our filter builder.  We need to restrict our results on edge search, as well
as on types, and any filters
      * that came from the grammar.
      */
-    private FilterBuilder createFilterBuilder( final SearchEdge searchEdge, final QueryVisitor
visitor, final
-    SearchTypes searchTypes ) {
+    private FilterBuilder createFilterBuilder( final SearchEdge searchEdge, final QueryVisitor
visitor,
+                                               final SearchTypes searchTypes ) {
         String context = createContextName( applicationScope, searchEdge );
 
 
@@ -227,7 +219,7 @@ public class SearchRequestBuilderStrategy {
         final String[] sourceTypes = searchTypes.getTypeNames( applicationScope );
 
 
-        if(sourceTypes.length > 0 ) {
+        if ( sourceTypes.length > 0 ) {
             final FilterBuilder[] typeTerms = new FilterBuilder[sourceTypes.length];
 
             for ( int i = 0; i < sourceTypes.length; i++ ) {
@@ -270,7 +262,6 @@ public class SearchRequestBuilderStrategy {
     }
 
 
-
     /**
      * Create a sort for the property name and field name specified
      *
@@ -284,7 +275,6 @@ public class SearchRequestBuilderStrategy {
         final TermFilterBuilder propertyFilter = sortPropertyTermFilter( propertyName );
 
 
-        return SortBuilders.fieldSort( fieldName ).order( sortOrder )
-                           .setNestedFilter( propertyFilter );
+        return SortBuilders.fieldSort( fieldName ).order( sortOrder ).setNestedFilter( propertyFilter
);
     }
 }


Mime
View raw message