usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mru...@apache.org
Subject [01/50] [abbrv] usergrid git commit: Changes to get Collections module tests running reliably, plus a new U.V. performance test like the one in the POC.
Date Mon, 11 Jul 2016 15:35:11 GMT
Repository: usergrid
Updated Branches:
  refs/heads/release-2.1.1 f95bf96cd -> 4f8183ff1


Changes to get Collections module tests running reliably, plus a new U.V. performance test
like the one in the POC.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8c6387f8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8c6387f8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8c6387f8

Branch: refs/heads/release-2.1.1
Commit: 8c6387f84fabcc86667f947502772402c979bdbb
Parents: 02c1795
Author: Dave Johnson <snoopdave@apache.org>
Authored: Tue Apr 26 12:49:52 2016 -0400
Committer: Dave Johnson <snoopdave@apache.org>
Committed: Tue Apr 26 12:49:52 2016 -0400

----------------------------------------------------------------------
 .../mvcc/stage/write/WriteUniqueVerify.java     |   2 +-
 .../mvcc/stage/write/WriteUniqueVerifyIT.java   |   2 +-
 .../mvcc/stage/write/WriteUniqueVerifyTest.java |  10 +-
 .../uniquevalues/UniqueValuesServiceTest.java   |  19 ++-
 .../apache/usergrid/rest/UniqueValuesIT.java    |   4 +-
 .../rest/UniqueValuesPerformanceIT.java         | 167 +++++++++++++++++++
 6 files changed, 188 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/8c6387f8/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 4fb8c39..e30ccd8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -105,7 +105,7 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
 
     @Override
     public void call( final CollectionIoEvent<MvccEntity> ioevent ) {
-        if ( akkaFig.getAkkaEnabled() ) {
+        if ( akkaFig != null && akkaFig.getAkkaEnabled() ) {
             verifyUniqueFieldsAkka( ioevent );
         } else {
             verifyUniqueFields( ioevent );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/8c6387f8/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
index f9ad211..ee610a9 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
@@ -76,7 +76,7 @@ public class WriteUniqueVerifyIT {
         if ( !startedAkka.getAndSet( true ) ) {
             ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization"
) );
             EntityCollectionManager manager = factory.createCollectionManager( context );
-            manager.startAkkaForTesting( "127.0.0.1", 2551, "us-east" );
+            manager.startAkkaForTesting( "127.0.0.1", 2552, "us-east" );
         }
     }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/8c6387f8/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
index 827212a..196f531 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
@@ -68,8 +68,6 @@ public class WriteUniqueVerifyTest {
     @Rule
     public MigrationManagerRule migrationManagerRule;
 
-
-
     @Inject
     private SerializationFig fig;
 
@@ -84,13 +82,14 @@ public class WriteUniqueVerifyTest {
         if ( !startedAkka.getAndSet( true ) ) {
             ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization"
) );
             EntityCollectionManager manager = factory.createCollectionManager( context );
-            manager.startAkkaForTesting( "127.0.0.1", 2551, "us-east" );
+            manager.startAkkaForTesting( "127.0.0.1", 2553, "us-east" );
         }
     }
 
 
     @Test
     public void testNoFields() throws ConnectionException {
+
         final ApplicationScope collectionScope = mock( ApplicationScope.class );
         final Keyspace keyspace = mock(Keyspace.class);
         final MutationBatch batch = mock(MutationBatch.class);
@@ -103,10 +102,9 @@ public class WriteUniqueVerifyTest {
         final MvccEntity mvccEntity = fromEntity( entity );
 
         // run the stage
-        WriteUniqueVerify newStage = new WriteUniqueVerify( uvstrat, fig, keyspace,cassandraConfig,
null, null );
+        WriteUniqueVerify newStage = new WriteUniqueVerify( uvstrat, fig, keyspace, cassandraConfig,
null, null );
 
-       newStage.call(
-            new CollectionIoEvent<>( collectionScope, mvccEntity ) ) ;
+       newStage.call( new CollectionIoEvent<>( collectionScope, mvccEntity ) ) ;
 
        //if we get here, it's a success.  We want to test no exceptions are thrown
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/8c6387f8/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
index c6efe80..341f604 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
@@ -17,6 +17,7 @@ import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
 import org.apache.usergrid.persistence.model.field.StringField;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -28,6 +29,7 @@ import java.util.Collection;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
 
@@ -52,6 +54,16 @@ public class UniqueValuesServiceTest {
     @Inject
     UniqueValuesService uniqueValuesService;
 
+    private static AtomicBoolean startedAkka = new AtomicBoolean( false );
+
+    @Before
+    public void initAkka() {
+        if ( !startedAkka.getAndSet( true ) ) {
+            ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization"
) );
+            EntityCollectionManager manager = factory.createCollectionManager( context );
+            manager.startAkkaForTesting( "127.0.0.1", 2554, "us-east" );
+        }
+    }
 
 
     /**
@@ -60,12 +72,7 @@ public class UniqueValuesServiceTest {
     @Test
     public void testDuplicatePrevention() throws Exception {
 
-        if ( !akkaFig.getAkkaEnabled() ) {
-            logger.warn("Skipping test because Akka is not enabled");
-            return;
-        }
-        uniqueValuesService.start("127.0.0.1", 2551, "us-east");
-        uniqueValuesService.waitForRequestActors();
+        initAkka();
 
         final AtomicInteger successCounter = new AtomicInteger( 0 );
         final AtomicInteger errorCounter = new AtomicInteger( 0 );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/8c6387f8/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueValuesIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueValuesIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueValuesIT.java
index 6303e89..c1bab62 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueValuesIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueValuesIT.java
@@ -81,7 +81,7 @@ public class UniqueValuesIT {
 
         final AtomicBoolean failed = new AtomicBoolean(false);
 
-        String randomizer = RandomStringUtils.randomAlphanumeric( 20 );
+        String randomizer = RandomStringUtils.randomAlphanumeric( 8 );
 
         String[] targetHosts = {"http://localhost:9090","http://localhost:9090"};
 
@@ -94,7 +94,7 @@ public class UniqueValuesIT {
 
                 if ( failed.get() ) { break; }
 
-                String username = "user_" + randomizer + "_" + i;
+                String username = "uv_test_user_" + randomizer;
                 final String host = targetHosts[ j % targetHosts.length ];
 
                 execService.submit( () -> {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/8c6387f8/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueValuesPerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueValuesPerformanceIT.java
b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueValuesPerformanceIT.java
new file mode 100644
index 0000000..cb57598
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueValuesPerformanceIT.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Form;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.text.DecimalFormat;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static com.codahale.metrics.MetricRegistry.name;
+
+
+/**
+ * Unique values performance test, not intended for use in normal JUnit testing.
+ *
+ * Just tests how fast we can create users, optionally against multiple end-points.
+ * (for multi-region testing).
+ */
+public class UniqueValuesPerformanceIT {
+    private static final Logger logger = LoggerFactory.getLogger( UniqueValuesPerformanceIT.class
);
+
+    private static final AtomicInteger successCounter = new AtomicInteger( 0 );
+    private static final AtomicInteger errorCounter = new AtomicInteger( 0 );
+
+
+    @Test
+    @Ignore("Intended for use against  prod-like cluster")
+    public void testBasicOperation() throws Exception {
+
+        int numUsers = 1000;
+        int numThreads = 1000;
+        int poolsize = 50;
+
+        ExecutorService execService = Executors.newFixedThreadPool( poolsize );
+
+        Multimap<String, Form> usersCreated =
+                Multimaps.synchronizedListMultimap( ArrayListMultimap.create( numUsers, 1
) );
+
+        Client client = ClientBuilder.newClient();
+
+        String randomizer = RandomStringUtils.randomAlphanumeric( 8 );
+
+        String[] targetHosts = {"http://localhost:9090","http://localhost:9090"};
+
+        final MetricRegistry metrics = new MetricRegistry();
+        final Timer responses = metrics.timer( name( UniqueValuesPerformanceIT.class, "responses"
) );
+        long startTime = System.currentTimeMillis();
+
+        for (int i = 0; i < numThreads; i++) {
+
+            execService.submit( () -> {
+
+                for (int j = 0; j < numUsers / numThreads; j++) {
+
+                    // every user gets unique name, no duplicates in this test
+                    UUID uuid = UUID.randomUUID();
+                    String username = "uv_test_user_" + randomizer;
+
+                    Form form = new Form();
+                    form.param( "name", username );
+                    form.param( "username", username );
+                    form.param( "email", username + "@example.org" );
+                    form.param( "password", "s3cr3t" );
+
+                    Timer.Context time = responses.time();
+                    try {
+                        final String host = targetHosts[j % targetHosts.length];
+                        WebTarget target = client.target( host ).path( "/management/users"
);
+
+                        Response response = target.request()
+                            .post( Entity.entity( form, MediaType.APPLICATION_FORM_URLENCODED
));
+
+                        if (response.getStatus() == 200 || response.getStatus() == 201) {
+                            usersCreated.put( username, form);
+                            successCounter.incrementAndGet();
+
+                        } else {
+                            String responseBody = response.readEntity( String.class );
+                            int rootCause = responseBody.indexOf( "root cause" );
+                            responseBody = rootCause > 0 ? responseBody.substring( rootCause,
rootCause + 100 ) : "";
+                            logger.error( "User creation failed status {} - {}", response.getStatus(),
responseBody );
+                            errorCounter.incrementAndGet();
+                        }
+
+                    } catch (Exception e) {
+                        errorCounter.incrementAndGet();
+                        logger.error( "Error", e );
+                    }
+                    time.stop();
+                }
+
+            } );
+        }
+        execService.shutdown();
+
+        try {
+            while (!execService.awaitTermination( 30, TimeUnit.SECONDS )) {
+                System.out.println( "Waiting..." );
+            }
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        }
+
+        long endTime = System.currentTimeMillis();
+
+        logger.info( "Total time {}s", (endTime - startTime) / 1000 );
+
+        DecimalFormat format = new DecimalFormat("##.###");
+
+        logger.info( "Timed {} requests:\n" +
+                "mean rate {}/s\n" +
+                "min       {}s\n" +
+                "max       {}s\n" +
+                "mean      {}s",
+                responses.getCount(),
+                format.format( responses.getMeanRate() ),
+                format.format( (double)responses.getSnapshot().getMin()  / 1000000000 ),
+                format.format( (double)responses.getSnapshot().getMax()  / 1000000000 ),
+                format.format( responses.getSnapshot().getMean()         / 1000000000 )
+        );
+
+        logger.info( "Error count {} ratio = {}",
+                errorCounter.get(), (float) errorCounter.get() / (float) responses.getCount()
);
+
+        logger.info( "Success count = {}",
+                successCounter.get() );
+
+
+        Assert.assertEquals( 0, errorCounter.get() );
+        Assert.assertEquals( numUsers, successCounter.get() );
+    }
+}


Mime
View raw message