usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject [04/35] usergrid git commit: Copy in just the tools changes from the master branch.
Date Mon, 25 Jan 2016 21:17:56 GMT
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
index f39ef9b..7a925b0 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
@@ -17,20 +17,20 @@
 package org.apache.usergrid.tools;
 
 
-import com.sun.org.apache.bcel.internal.generic.DUP;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 import org.apache.commons.io.filefilter.PrefixFileFilter;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.management.OrganizationInfo;
 import org.apache.usergrid.management.UserInfo;
 import org.apache.usergrid.persistence.EntityManager;
 import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Identifier;
 import org.apache.usergrid.persistence.SimpleEntityRef;
 import org.apache.usergrid.persistence.entities.User;
 import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.apache.usergrid.persistence.index.query.Identifier;
 import org.codehaus.jackson.JsonFactory;
 import org.codehaus.jackson.JsonParser;
 import org.codehaus.jackson.JsonToken;
@@ -39,7 +39,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
-import java.util.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
@@ -47,18 +50,16 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE;
 import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
-
 
 /**
  * Import Admin Users and metadata including organizations and passwords.
- * 
- * Usage Example: 
- * 
+ *
+ * Usage Example:
+ *
  * java -Xmx8000m -Dlog4j.configuration=file:/home/me/log4j.properties -classpath . \
  *      -jar usergrid-tools-1.0.2.jar ImportAdmins -writeThreads 100 -auditThreads 100 \
- *      -host casshost -inputDir=/home/me/import-data 
- *      
+ *      -host casshost -inputDir=/home/me/import-data
+ *
  * If you want to provide any property overrides, put properties file named usergrid-custom-tools.properties
  * in the same directory where you run the above command. For example, you might want to set the Cassandra
  * client threads and import to a specific set of keyspaces:
@@ -97,8 +98,8 @@ public class ImportAdmins extends ToolBase {
     AtomicInteger writeEmptyCount = new AtomicInteger( 0 );
     AtomicInteger auditEmptyCount = new AtomicInteger( 0 );
     AtomicInteger metadataEmptyCount = new AtomicInteger( 0 );
-    
-    
+
+
     static class DuplicateUser {
         String email;
         String username;
@@ -110,7 +111,7 @@ public class ImportAdmins extends ToolBase {
             }
         }
     }
-    
+
 
 
     @Override
@@ -269,7 +270,7 @@ public class ImportAdmins extends ToolBase {
             long timeRemaining = (long) ( sizeLast / (rateAverageSum / iterations) );
 
             logger.info("++PROGRESS ({}): sizeLast={} nowSize={} processed={} rateLast={}/s rateAvg={}/s timeRemaining={}s",
-                new Object[] { 
+                new Object[] {
                     identifier, sizeLast, sizeNow, processed, rateLast, (rateAverageSum / iterations), timeRemaining } );
         }
 
@@ -355,7 +356,7 @@ public class ImportAdmins extends ToolBase {
             workerThread.start();
             metadataWorkerThreadMap.put(worker, workerThread);
         }
-        
+
         logger.info( "Started {} metadata workers", writeThreadCount );
     }
 
@@ -363,7 +364,7 @@ public class ImportAdmins extends ToolBase {
     @SuppressWarnings("unchecked")
     private void importMetadata(String fileName, int writeThreads) throws Exception {
 
-        EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+        EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID);
 
         File metadataFile = new File(importDir, fileName);
 
@@ -401,10 +402,10 @@ public class ImportAdmins extends ToolBase {
                 try {
                     EntityRef entityRef = new SimpleEntityRef( "user", UUID.fromString( entityOwnerId ) );
                     Map<String, Object> metadata = (Map<String, Object>) jp.readValueAs( Map.class );
-                    
+
                     workQueue.put( new ImportMetadataTask( entityRef, metadata ) );
                     logger.debug( "Put user {} in metadata queue", entityRef.getUuid() );
-                    
+
                 } catch ( Exception e ) {
                     logger.debug( "Error with user {}, not putting in metadata queue", entityOwnerId );
                 }
@@ -426,7 +427,7 @@ public class ImportAdmins extends ToolBase {
             EntityManager em, EntityRef entityRef, Map<String, Object> metadata) throws Exception {
 
         DuplicateUser dup = dupsByDupUuid.get( entityRef.getUuid() );
-        
+
         if ( dup == null ) { // not a duplicate
 
             User user = em.get( entityRef, User.class );
@@ -476,7 +477,7 @@ public class ImportAdmins extends ToolBase {
                     }
                 }
             }
-            
+
             Map<String, Object> dictionariesMap = (Map<String, Object>) metadata.get("dictionaries");
             if (dictionariesMap != null && !dictionariesMap.isEmpty()) {
                 for (String name : dictionariesMap.keySet()) {
@@ -501,17 +502,17 @@ public class ImportAdmins extends ToolBase {
             } else {
                 logger.warn("User {} has no dictionaries", entityRef.getUuid() );
             }
-            
+
         } else { // this is a duplicate user, so merge orgs
 
             logger.info("Processing duplicate username={} email={}", dup.email, dup.username );
-           
-            Identifier identifier = dup.email != null ? 
+
+            Identifier identifier = dup.email != null ?
                 Identifier.fromEmail( dup.email ) : Identifier.from( dup.username );
             User originalUser = em.get( em.getUserByIdentifier(identifier), User.class );
 
             // get map of original user's orgs
-            
+
             UserInfo originalUserInfo = managementService.getAdminUserByEmail( originalUser.getEmail() );
             Map<String, Object> originalUserOrgData =
                     managementService.getAdminUserOrganizationData( originalUser.getUuid() );
@@ -530,7 +531,7 @@ public class ImportAdmins extends ToolBase {
                 if (originalUserOrgs.get( orgName ) == null) { // original user does not have this org
 
                     OrganizationInfo orgInfo = managementService.getOrganizationByUuid( orgUuid );
-                    
+
                     if (orgInfo == null) { // org does not exist yet, create it and add original user to it
                         try {
                             managementService.createOrganization( orgUuid, orgName, originalUserInfo, false );
@@ -539,9 +540,9 @@ public class ImportAdmins extends ToolBase {
                             logger.debug( "Created new org {} for user {}:{}:{} from duplicate user {}:{}",
                                 new Object[]{
                                         orgInfo.getName(),
-                                        originalUser.getUsername(), 
+                                        originalUser.getUsername(),
                                         originalUser.getEmail(),
-                                        originalUser.getUuid(), 
+                                        originalUser.getUuid(),
                                         dup.username, dup.email
                                 });
 
@@ -554,20 +555,20 @@ public class ImportAdmins extends ToolBase {
                             logger.debug( "Added to org user {}:{}:{} from duplicate user {}:{}",
                                     new Object[]{
                                             orgInfo.getName(),
-                                            originalUser.getUsername(), 
+                                            originalUser.getUsername(),
                                             originalUser.getEmail(),
-                                            originalUser.getUuid(), 
+                                            originalUser.getUuid(),
                                             dup.username, dup.email
                                     });
 
                         } catch (Exception e) {
-                            logger.error( "Error Adding user {} to org {}", 
+                            logger.error( "Error Adding user {} to org {}",
                                     new Object[]{originalUserInfo.getEmail(), orgName} );
                         }
                     }
-                    
+
                 } // else original user already has this org
-                
+
             }
         }
     }
@@ -612,7 +613,7 @@ public class ImportAdmins extends ToolBase {
         public void run() {
             int count = 0;
 
-            EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+            EntityManager em = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
 
             long durationSum = 0;
 
@@ -647,7 +648,7 @@ public class ImportAdmins extends ToolBase {
                     durationSum += duration;
 
                     //logger.debug( "Audited {}th admin", userCount );
-                    
+
                     if ( count % 100 == 0 ) {
                         logger.info( "Audited {}. Average Audit Rate: {}(ms)", count, durationSum / count );
                     }
@@ -690,7 +691,7 @@ public class ImportAdmins extends ToolBase {
         public void run() {
             int count = 0;
 
-            EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+            EntityManager em = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
 
             long durationSum = 0;
 
@@ -705,20 +706,20 @@ public class ImportAdmins extends ToolBase {
                         continue;
                     }
                     metadataEmptyCount.set( 0 );
-                    
+
                     long startTime = System.currentTimeMillis();
-                    
+
                     importEntityMetadata( em, task.entityRef, task.metadata );
-                    
+
                     long stopTime = System.currentTimeMillis();
                     long duration = stopTime - startTime;
                     durationSum += duration;
                     metadataCount.getAndIncrement();
                     count++;
-                    
+
                     if ( count % 30 == 0 ) {
                         logger.info( "Imported {} metadata of total {} expected. " +
-                                        "Average metadata Imported Rate: {}(ms)", 
+                                        "Average metadata Imported Rate: {}(ms)",
                            new Object[] { metadataCount.get(), userCount.get(), durationSum / count });
                     }
 
@@ -752,7 +753,7 @@ public class ImportAdmins extends ToolBase {
         public void run() {
             int count = 0;
 
-            EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+            EntityManager em = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
 
             long durationSum = 0;
 
@@ -776,7 +777,7 @@ public class ImportAdmins extends ToolBase {
 
                     try {
                         long startTime = System.currentTimeMillis();
-                        
+
                         em.create(uuid, type, entityProps);
 
                         logger.debug( "Imported admin user {}:{}:{}",
@@ -787,20 +788,20 @@ public class ImportAdmins extends ToolBase {
                         long stopTime = System.currentTimeMillis();
                         long duration = stopTime - startTime;
                         durationSum += duration;
-                        
+
                         count++;
                         if (count % 30 == 0) {
                             logger.info( "This worked has imported {} users of total {} imported so far. " +
-                                            "Average Creation Rate: {}ms", 
+                                            "Average Creation Rate: {}ms",
                                 new Object[] { count, userCount.get(), durationSum / count });
                         }
-                        
+
                     } catch (DuplicateUniquePropertyExistsException de) {
                         String dupProperty = de.getPropertyName();
                         handleDuplicateAccount( em, dupProperty, entityProps );
                         continue;
 
-                        
+
                     } catch (Exception e) {
                         logger.error("Error", e);
                     }
@@ -811,19 +812,19 @@ public class ImportAdmins extends ToolBase {
             }
         }
 
-        
+
         private void handleDuplicateAccount(EntityManager em, String dupProperty, Map<String, Object> entityProps ) {
 
             logger.info( "Processing duplicate user {}:{}:{} with duplicate {}", new Object[]{
-                    entityProps.get( "username" ), 
-                    entityProps.get( "email" ), 
-                    entityProps.get( "uuid" ), 
+                    entityProps.get( "username" ),
+                    entityProps.get( "email" ),
+                    entityProps.get( "uuid" ),
                     dupProperty} );
-           
+
             UUID dupUuid = UUID.fromString( entityProps.get("uuid").toString() );
             try {
                 dupsByDupUuid.put( dupUuid, new DuplicateUser( dupProperty, entityProps ) );
-                
+
             } catch (Exception e) {
                 logger.error("Error processing dup user {}:{}:{}",
                         new Object[] {entityProps.get( "username" ), entityProps.get("email"), dupUuid});

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java b/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
deleted file mode 100644
index 5c35e85..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.EntityRef;
-
-
-/**
- * Index rebuild utility for Usergrid. Can be used to rebuild the index for a specific 
- * application, a specific application's collection or for an entire Usergrid system.
- */
-public class IndexRebuild extends ToolBase {
-
-    private static final String APPLICATION_ARG = "app";
-
-    private static final String COLLECTION_ARG = "col";
-
-    private static final String ALL_ARG = "all";
-
-    private static final int PAGE_SIZE = 100;
-
-
-    private static final Logger logger = LoggerFactory.getLogger( IndexRebuild.class );
-
-
-    @Override
-    @SuppressWarnings("static-access")
-    public Options createOptions() {
-
-        Option hostOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
-                .withDescription( "Cassandra host" ).create( "host" );
-
-        Option esHostsOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
-                .withDescription( "ElasticSearch host" ).create( "eshost" );
-
-        Option esClusterOpt = OptionBuilder.withArgName( "host" ).hasArg().isRequired( true )
-                .withDescription( "ElasticSearch cluster name" ).create( "escluster" );
-
-        Option appOpt = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
-                .withDescription( "Application id or app name" ).create( APPLICATION_ARG );
-
-        Option collOpt = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
-                .withDescription( "Collection name" ).create( COLLECTION_ARG );
-
-        Option allOpt = OptionBuilder.withType( Boolean.class )
-                .withArgName( ALL_ARG ).hasArg().isRequired( false )
-                .withDescription( "True to reindex all application" ).create( ALL_ARG );
-
-        Options options = new Options();
-        options.addOption( hostOpt );
-        options.addOption( esHostsOpt );
-        options.addOption( esClusterOpt );
-        options.addOption( appOpt );
-        options.addOption( collOpt );
-        options.addOption( allOpt );
-
-        return options;
-    }
-
-
-    /*
-     * (non-Javadoc)
-     * 
-     * @see
-     * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
-     */
-    @Override
-    public void runTool( CommandLine line ) throws Exception {
-        startSpring();
-
-        logger.info( "Starting index rebuild" );
-
-        EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
-            
-            @Override
-            public void onProgress(EntityRef entity) {
-                logger.info("Indexing entity {}:{}", entity.getType(), entity.getUuid());
-            }
-
-            @Override
-            public long getWriteDelayTime() {
-                return 100;
-            }
-        };
-
-        emf.rebuildInternalIndexes( po ); 
-        emf.refreshIndex();
-
-        if ( line.getOptionValue("all") != null && line.getOptionValue("all").equalsIgnoreCase("true") ) {
-            emf.rebuildAllIndexes( po );
-
-        } else if ( line.getOptionValue( APPLICATION_ARG ) != null ) {
-
-            // Goes through each app id specified
-            for ( UUID appId : getAppIds( line ) ) {
-
-                logger.info( "Reindexing for app id: {}", appId );
-                Set<String> collections = getCollections( line, appId );
-
-                for ( String collection : collections ) {
-                    emf.rebuildCollectionIndex( appId, collection, po );
-                    emf.refreshIndex();
-                }
-            }
-
-        } else {
-
-            Map<String, UUID> ids = emf.getApplications();
-            System.out.println( "Printing all apps" );
-            for ( Entry<String, UUID> entry : ids.entrySet() ) {
-                System.out.println( entry.getKey() + " appid=" + entry.getValue() );
-            }
-        }
-
-        logger.info( "Finished index rebuild" );
-    }
-
-
-    /** Get all app id */
-    private Collection<UUID> getAppIds( CommandLine line ) throws Exception {
-
-        String appId = line.getOptionValue( APPLICATION_ARG );
-
-        Map<String, UUID> ids = emf.getApplications();
-
-        if ( appId != null ) {
-            UUID id = UUIDUtils.tryExtractUUID( appId );
-            if ( id == null ) {
-                logger.debug("Got applications: " + ids );
-                id = emf.getApplications().get( appId );
-            }
-            return Collections.singleton( id );
-        }
-
-        return ids.values();
-    }
-
-
-    /** Get collection names. If none are specified, all are returned */
-    private Set<String> getCollections( CommandLine line, UUID appId ) throws Exception {
-
-        String passedName = line.getOptionValue( COLLECTION_ARG );
-
-        if ( passedName != null ) {
-            return Collections.singleton( passedName );
-        }
-
-        EntityManager em = emf.getEntityManager( appId );
-
-        return em.getApplicationCollections();
-    }
-}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java
index b308290..a056f77 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java
@@ -26,6 +26,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 
+import org.apache.usergrid.persistence.index.query.CounterResolution;
 import org.codehaus.jackson.JsonGenerator;
 import org.apache.usergrid.management.ApplicationInfo;
 import org.apache.usergrid.management.OrganizationInfo;
@@ -46,7 +47,6 @@ import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.BiMap;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Ordering;
-import org.apache.usergrid.persistence.index.query.CounterResolution;
 
 
 /**

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java b/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java
index 4fbccb2..4493b67 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java
@@ -25,7 +25,7 @@ import java.util.UUID;
 import org.apache.usergrid.management.UserInfo;
 import org.apache.usergrid.persistence.Entity;
 import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.Query;
 import org.apache.usergrid.persistence.Results;
 
 import org.apache.commons.cli.CommandLine;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java b/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java
index b67e9eb..48e2af6 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java
@@ -46,7 +46,7 @@ import static org.apache.usergrid.services.ServicePayload.payload;
 
 public class PopulateSample extends ToolBase {
 
-    private static final Logger logger = LoggerFactory.getLogger( Export.class );
+    private static final Logger logger = LoggerFactory.getLogger( PopulateSample.class );
 
 
     @Override

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/RepairingMismatchedApplicationMetadata.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/RepairingMismatchedApplicationMetadata.java b/stack/tools/src/main/java/org/apache/usergrid/tools/RepairingMismatchedApplicationMetadata.java
deleted file mode 100644
index 7b18ccc..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/RepairingMismatchedApplicationMetadata.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Options;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-
-import org.apache.usergrid.management.OrganizationInfo;
-
-import com.google.common.collect.BiMap;
-
-import me.prettyprint.cassandra.serializers.ByteBufferSerializer;
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.mutation.Mutator;
-
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.batchExecute;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.APPLICATIONS_CF;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.RETRY_COUNT;
-
-
-public class RepairingMismatchedApplicationMetadata extends ToolBase {
-
-    public static final ByteBufferSerializer be = new ByteBufferSerializer();
-
-    private static final Logger logger = LoggerFactory.getLogger( RepairingMismatchedApplicationMetadata.class );
-
-
-    @Override
-    public Options createOptions() {
-        Options options = super.createOptions();
-        return options;
-    }
-
-
-    @Override
-    public void runTool( CommandLine line ) throws Exception {
-        startSpring();
-
-        //sucks, but it's not picking up the configuration
-        LogManager.getLogger( RepairingMismatchedApplicationMetadata.class ).setLevel( Level.INFO );
-
-        UUID orgId = null;
-        List<OrganizationInfo> orgs;
-
-        final int size = 1000;
-
-
-        do {
-            orgs = managementService.getOrganizations( orgId, size );
-
-
-            for ( OrganizationInfo org : orgs ) {
-
-                orgId = org.getUuid();
-
-                logger.info( "Auditing org {}", org.getName() );
-
-                try {
-                    BiMap<UUID, String> apps = managementService.getApplicationsForOrganization( org.getUuid() );
-
-
-                    for ( Map.Entry<UUID, String> app : apps.entrySet() ) {
-
-                        logger.info( "Auditing org {} app {}", org.getName(), app.getValue() );
-
-                        UUID applicationId = emf.lookupApplication( app.getValue() );
-                        if ( applicationId == null ) {
-                            String appName = app.getValue();
-                            Keyspace ko = cass.getSystemKeyspace();
-                            Mutator<ByteBuffer> m = createMutator( ko, be );
-                            long timestamp = cass.createTimestamp();
-                            addInsertToMutator( m, APPLICATIONS_CF, appName, PROPERTY_UUID, app.getKey(), timestamp );
-                            addInsertToMutator( m, APPLICATIONS_CF, appName, PROPERTY_NAME, appName, timestamp );
-                            batchExecute( m, RETRY_COUNT );
-                            logger.info( "Repairing alias with app uuid {}, and name {}", app.getKey(),
-                                    app.getValue() );
-                        }
-                    }
-                }
-                catch ( Exception e ) {
-                    logger.error( "Unable to process applications for organization {}", org, e );
-                }
-            }
-        }
-        while ( orgs != null && orgs.size() == size );
-
-        logger.info( "Completed repairing aliases" );
-        Thread.sleep( 1000 * 60 );
-    }
-}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
index aa33e49..7fb7015 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
@@ -101,6 +101,7 @@ public abstract class ToolBase {
             System.setProperty( "cassandra.url", line.getOptionValue( "host" ) );
             System.setProperty( "elasticsearch.hosts", line.getOptionValue( "eshost" ) );
             System.setProperty( "elasticsearch.cluster_name", line.getOptionValue( "escluster" ) );
+            System.setProperty( "usergrid.cluster_name", "usergrid" );
         }
 
         try {
@@ -134,6 +135,12 @@ public abstract class ToolBase {
         Option hostOption = OptionBuilder.withArgName( "host" ).hasArg()
             .withDescription( "Cassandra host" ).create( "host" );
 
+        Option esHostOption = OptionBuilder.withArgName( "eshost" ).hasArg()
+            .withDescription( "ElasticSearch host" ).create( "eshost" );
+
+        Option esClusterOption = OptionBuilder.withArgName( "escluster" ).hasArg()
+            .withDescription( "ElasticSearch cluster name" ).create( "escluster" );
+
         Option remoteOption = OptionBuilder
             .withDescription( "Use remote Cassandra instance" ).create( "remote" );
 
@@ -143,6 +150,8 @@ public abstract class ToolBase {
 
         Options options = new Options();
         options.addOption( hostOption );
+        options.addOption( esHostOption );
+        options.addOption( esClusterOption );
         options.addOption( remoteOption );
         options.addOption( verbose );
 
@@ -181,7 +190,7 @@ public abstract class ToolBase {
 
         Setup setup = ( (CpEntityManagerFactory) emf ).getSetup();
         logger.info( "Setting up Usergrid schema" );
-        setup.init();
+        setup.initSubsystems();
         logger.info( "Usergrid schema setup" );
 
         logger.info( "Setting up Usergrid management services" );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
deleted file mode 100644
index 0ae45d6..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ /dev/null
@@ -1,386 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.management.ApplicationInfo;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.index.query.Identifier;
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
-import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
-import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
-import org.apache.usergrid.persistence.entities.Application;
-import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
-import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
-import org.apache.usergrid.persistence.query.ir.result.UUIDIndexSliceParser;
-import org.apache.usergrid.persistence.schema.CollectionInfo;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.AbstractComposite.ComponentEquality;
-import me.prettyprint.hector.api.beans.DynamicComposite;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.mutation.Mutator;
-
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_COLLECTIONS;
-import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
-import org.apache.usergrid.persistence.SimpleEntityRef;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX_ENTRIES;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.INDEX_ENTRY_LIST_COUNT;
-import static org.apache.usergrid.utils.CompositeUtils.setEqualityFlag;
-import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
-import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
-
-
-/**
- * This is a utility to audit all available entity ids in the secondary index. It then checks to see if any index value
- * is not present in the Entity_Index_Entries. If it is not, the value from the index is removed, and a forced re-index
- * is triggered
- * <p/>
- * USERGRID-323
- * <p/>
- * <p/>
- * UniqueIndexCleanup -app [appid] -col [collectionname]
- *
- * @author tnine
- */
-public class UniqueIndexCleanup extends ToolBase {
-
-    /**
-     *
-     */
-    private static final int PAGE_SIZE = 100;
-
-
-
-    private static final Logger logger = LoggerFactory.getLogger( UniqueIndexCleanup.class );
-
-    /**
-     *
-     */
-    private static final String APPLICATION_ARG = "app";
-
-    /**
-     *
-     */
-    private static final String COLLECTION_ARG = "col";
-
-
-    @Override
-    @SuppressWarnings("static-access")
-    public Options createOptions() {
-
-
-        Options options = new Options();
-
-        Option hostOption =
-                OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
-                             .create( "host" );
-
-
-        options.addOption( hostOption );
-
-
-        Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
-                                        .withDescription( "application id or app name" ).create( APPLICATION_ARG );
-
-
-        options.addOption( appOption );
-
-        Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
-                                               .withDescription( "colleciton name" ).create( COLLECTION_ARG );
-
-        options.addOption( collectionOption );
-
-        return options;
-    }
-
-
-    /*
-     * (non-Javadoc)
-     *
-     * @see
-     * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
-     */
-    @Override
-    public void runTool( CommandLine line ) throws Exception {
-        startSpring();
-
-        logger.info( "Starting entity cleanup" );
-
-        Map<String, UUID> apps = getApplications( emf, line );
-
-
-        for ( Entry<String, UUID> app : apps.entrySet() ) {
-
-            logger.info( "Starting cleanup for app {}", app.getKey() );
-
-            UUID applicationId = app.getValue();
-            EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
-
-            //sanity check for corrupt apps
-            Application appEntity = em.getApplication();
-
-            if ( appEntity == null ) {
-                logger.warn( "Application does not exist in data. {}", app.getKey() );
-                continue;
-            }
-
-            CassandraService cass = em.getCass();
-            IndexBucketLocator indexBucketLocator = em.getIndexBucketLocator();
-
-            Keyspace ko = cass.getApplicationKeyspace( applicationId );
-
-            UUID timestampUuid = newTimeUUID();
-            long timestamp = getTimestampInMicros( timestampUuid );
-
-
-            // go through each collection and audit the values
-            for ( String collectionName : getCollectionNames( em, line ) ) {
-
-
-                IndexScanner scanner = cass.getIdList( cass.getApplicationKeyspace( applicationId ),
-                        key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null, PAGE_SIZE, false,
-                        indexBucketLocator, applicationId, collectionName, false );
-
-                SliceIterator itr = new SliceIterator( null, scanner, new UUIDIndexSliceParser() );
-
-
-                while ( itr.hasNext() ) {
-
-                    Set<ScanColumn> ids = itr.next();
-
-                    CollectionInfo collection = getDefaultSchema().getCollection( "application", collectionName );
-
-
-                    //We shouldn't have to do this, but otherwise the cursor won't work
-                    Set<String> indexed = collection.getPropertiesIndexed();
-
-                    // what's left needs deleted, do so
-
-                    logger.info( "Auditing {} entities for collection {} in app {}", new Object[] {
-                            ids.size(), collectionName, app.getValue()
-                    } );
-
-                    for ( ScanColumn col : ids ) {
-                        final UUID id = col.getUUID();
-                        String type = getDefaultSchema().getCollectionType("application", collectionName);
-
-                        boolean reIndex = false;
-
-                        Mutator<ByteBuffer> m = createMutator( ko, be );
-
-                        try {
-
-                            for ( String prop : indexed ) {
-
-                                String bucket =
-                                        indexBucketLocator.getBucket( applicationId, IndexType.COLLECTION, id, prop );
-
-                                Object rowKey = key( applicationId, collection.getName(), prop, bucket );
-
-                                List<HColumn<ByteBuffer, ByteBuffer>> indexCols =
-                                        scanIndexForAllTypes( ko, indexBucketLocator, applicationId, rowKey, id, prop );
-
-                                // loop through the indexed values and verify them as present in
-                                // our entity_index_entries. If they aren't, we need to delete the
-                                // from the secondary index, and mark
-                                // this object for re-index via n update
-                                for ( HColumn<ByteBuffer, ByteBuffer> index : indexCols ) {
-
-                                    DynamicComposite secondaryIndexValue =
-                                            DynamicComposite.fromByteBuffer( index.getName().duplicate() );
-
-                                    Object code = secondaryIndexValue.get( 0 );
-                                    Object propValue = secondaryIndexValue.get( 1 );
-                                    UUID timestampId = ( UUID ) secondaryIndexValue.get( 3 );
-
-                                    DynamicComposite existingEntryStart =
-                                            new DynamicComposite( prop, code, propValue, timestampId );
-                                    DynamicComposite existingEntryFinish =
-                                            new DynamicComposite( prop, code, propValue, timestampId );
-
-                                    setEqualityFlag( existingEntryFinish, ComponentEquality.GREATER_THAN_EQUAL );
-
-                                    // now search our EntityIndexEntry for previous values, see if
-                                    // they don't match this one
-
-                                    List<HColumn<ByteBuffer, ByteBuffer>> entries =
-                                            cass.getColumns( ko, ENTITY_INDEX_ENTRIES, id, existingEntryStart,
-                                                    existingEntryFinish, INDEX_ENTRY_LIST_COUNT, false );
-
-                                    // we wouldn't find this column in our entity_index_entries
-                                    // audit. Delete it, then mark this entity for update
-                                    if ( entries.size() == 0 ) {
-                                        logger.info(
-                                            "Could not find reference to value '{}' property '{}'"+
-                                            " on entity {} in collection {}. " + " Forcing reindex",
-                                            new Object[] { propValue, prop, id, collectionName } );
-
-                                        addDeleteToMutator(
-                                            m, ENTITY_INDEX, rowKey, index.getName().duplicate(), timestamp );
-
-                                        reIndex = true;
-                                    }
-
-                                    if ( entries.size() > 1 ) {
-                                        logger.info(
-                                            "Found more than 1 entity referencing unique index "
-                                          + "for property '{}' with value " + "'{}'",
-                                            prop, propValue );
-                                        reIndex = true;
-                                    }
-                                }
-                            }
-
-                            //force this entity to be updated
-                            if ( reIndex ) {
-                                Entity entity = em.get( new SimpleEntityRef( type, id ));
-
-                                //entity may not exist, but we should have deleted rows from the index
-                                if ( entity == null ) {
-                                    logger.warn( "Entity with id {} did not exist in app {}",
-                                            id, applicationId );
-                                    //now execute the cleanup. In this case the entity is gone,
-                                    // so we'll want to remove references from
-                                    // the secondary index
-                                    m.execute();
-                                    continue;
-                                }
-
-
-                                logger.info( "Reindex complete for entity with id '{} ", id );
-                                em.update( entity );
-
-                                //now execute the cleanup. This way if the above update fails,
-                                // we still have enough data to run again
-                                // later
-                                m.execute();
-                            }
-                        }
-                        catch ( Exception e ) {
-                            logger.error( "Unable to process entity with id '{}'", id, e );
-                        }
-                    }
-                }
-            }
-        }
-
-        logger.info( "Completed audit of apps" );
-    }
-
-
-    private Map<String, UUID> getApplications( EntityManagerFactory emf, CommandLine line ) throws Exception {
-        String appName = line.getOptionValue( APPLICATION_ARG );
-
-        if ( appName == null ) {
-            return emf.getApplications();
-        }
-
-        ApplicationInfo app = managementService.getApplicationInfo( Identifier.from( appName ) );
-
-        if ( app == null ) {
-            logger.error( "Could not find application with id or name {}", appName );
-            System.exit( 3 );
-        }
-
-
-        Map<String, UUID> apps = new HashMap<String, UUID>();
-
-        apps.put( app.getName(), app.getId() );
-
-        return apps;
-    }
-
-
-    private Set<String> getCollectionNames( EntityManager em, CommandLine line ) throws Exception {
-
-        String collectionName = line.getOptionValue( COLLECTION_ARG );
-
-        if ( collectionName == null ) {
-            return em.getApplicationCollections();
-        }
-
-
-        Set<String> names = new HashSet<String>();
-        names.add( collectionName );
-
-        return names;
-    }
-
-
-    private List<HColumn<ByteBuffer, ByteBuffer>> scanIndexForAllTypes( Keyspace ko,
-                                                                        IndexBucketLocator indexBucketLocator,
-                                                                        UUID applicationId, Object rowKey,
-                                                                        UUID entityId, String prop ) throws Exception {
-
-        //TODO Determine the index bucket.  Scan the entire index for properties with this entityId.
-
-
-        DynamicComposite start = null;
-
-        List<HColumn<ByteBuffer, ByteBuffer>> cols;
-
-        List<HColumn<ByteBuffer, ByteBuffer>> results = new ArrayList<HColumn<ByteBuffer, ByteBuffer>>();
-
-
-        do {
-            cols = cass.getColumns( ko, ENTITY_INDEX, rowKey, start, null, 100, false );
-
-            for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
-                DynamicComposite secondaryIndexValue = DynamicComposite.fromByteBuffer( col.getName().duplicate() );
-
-                UUID storedId = ( UUID ) secondaryIndexValue.get( 2 );
-
-                //add it to the set.  We can't short circuit due to property ordering
-                if ( entityId.equals( storedId ) ) {
-                    results.add( col );
-                }
-
-                start = secondaryIndexValue;
-            }
-        }
-        while ( cols.size() == 100 );
-
-        return results;
-    }
-}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java
index 3b5383d..e4195a4 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.tools;
 
 
 import com.google.common.collect.BiMap;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.management.UserInfo;
 
 import org.apache.commons.cli.CommandLine;
@@ -65,7 +66,7 @@ public class UserManager extends ToolBase {
         logger.info( mapToFormattedJsonString( orgs ) );
 
         logger.info("--- User dictionaries:");
-        EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
+        EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
         User user = em.get( userInfo.getUuid(), User.class );
         Set<String> dictionaries = em.getDictionaries( user );
         for (String dictionary : dictionaries) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java b/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java
index 7b60257..48d47be 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java
@@ -17,63 +17,42 @@
 package org.apache.usergrid.tools;
 
 
+import au.com.bytecode.opencsv.CSVReader;
+import au.com.bytecode.opencsv.CSVWriter;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.PutObjectResult;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Module;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.entities.Application;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
+import org.apache.usergrid.utils.InflectionUtils;
+import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
+import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.jclouds.netty.config.NettyPayloadModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.io.File;
 import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.FilenameFilter;
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
-import org.jclouds.ContextBuilder;
-import org.jclouds.blobstore.AsyncBlobStore;
-import org.jclouds.blobstore.BlobStoreContext;
-import org.jclouds.blobstore.domain.Blob;
-import org.jclouds.blobstore.domain.BlobBuilder;
-import org.jclouds.blobstore.options.PutOptions;
-import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
-import org.jclouds.logging.log4j.config.Log4JLoggingModule;
-import org.jclouds.netty.config.NettyPayloadModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.Schema;
-import org.apache.usergrid.persistence.entities.Application;
-import org.apache.usergrid.persistence.schema.CollectionInfo;
-import org.apache.usergrid.utils.InflectionUtils;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-
-import com.google.common.collect.BiMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.inject.Module;
-
-import au.com.bytecode.opencsv.CSVReader;
-import au.com.bytecode.opencsv.CSVWriter;
-
 import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
-import org.apache.usergrid.persistence.index.query.Query.Level;
 
 
 /**
@@ -84,7 +63,7 @@ import org.apache.usergrid.persistence.index.query.Query.Level;
  */
 public class WarehouseExport extends ExportingToolBase {
 
-    private static final Logger LOG = LoggerFactory.getLogger( WarehouseExport.class );
+    private static final Logger logger = LoggerFactory.getLogger( WarehouseExport.class );
     private static final char SEPARATOR = '|';
 
     public static final String BUCKET_PROPNAME = "usergrid.warehouse-export-bucket";
@@ -136,14 +115,14 @@ public class WarehouseExport extends ExportingToolBase {
         applyOrgId( line );
         prepareBaseOutputFileName( line );
         outputDir = createOutputParentDir();
-        LOG.info( "Export directory: {}", outputDir.getAbsolutePath() );
+        logger.info( "Export directory: {}", outputDir.getAbsolutePath() );
 
         // create writer
         applyStartTime( line );
         applyEndTime( line );
-        LOG.error( "startTime: {}, endTime: {}", startTime, endTime );
+        logger.error( "startTime: {}, endTime: {}", startTime, endTime );
         if ( startTime.getTime() >= endTime.getTime() ) {
-            LOG.error( "startTime must be before endTime. exiting." );
+            logger.error( "startTime must be before endTime. exiting." );
             System.exit( 1 );
         }
 
@@ -175,7 +154,7 @@ public class WarehouseExport extends ExportingToolBase {
 
         // now that file is written, copy it to S3
         if ( line.hasOption( "upload" ) ) {
-            LOG.info( "Copy to S3" );
+            logger.info( "Copy to S3" );
             copyToS3( fileName );
         }
     }
@@ -195,40 +174,16 @@ public class WarehouseExport extends ExportingToolBase {
                 .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
                         new NettyPayloadModule() );
 
-        BlobStoreContext context =
-                ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules( MODULES )
-                              .overrides( overrides ).buildView( BlobStoreContext.class );
-
-        // Create Container (the bucket in s3)
-        try {
-            AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed to sync
-            // BlobStore (returns false if it already exists)
-            ListenableFuture<Boolean> container = blobStore.createContainerInLocation( null, bucketName );
-            if ( container.get() ) {
-                LOG.info( "Created bucket " + bucketName );
-            }
-        }
-        catch ( Exception ex ) {
-            logger.error( "Could not start binary service: {}", ex.getMessage() );
-            throw new RuntimeException( ex );
-        }
-
-        try {
-            File file = new File( fileName );
-            AsyncBlobStore blobStore = context.getAsyncBlobStore();
-            BlobBuilder blobBuilder =
-                    blobStore.blobBuilder( file.getName() ).payload( file ).calculateMD5().contentType( "text/plain" )
-                             .contentLength( file.length() );
-
-            Blob blob = blobBuilder.build();
+        AWSCredentials credentials = new BasicAWSCredentials(accessId, secretKey);
+        ClientConfiguration clientConfig = new ClientConfiguration();
+        clientConfig.setProtocol( Protocol.HTTP);
 
-            ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
+        AmazonS3Client s3Client = new AmazonS3Client(credentials, clientConfig);
 
-            LOG.info( "Uploaded file etag=" + futureETag.get() );
-        }
-        catch ( Exception e ) {
-            LOG.error( "Error uploading to blob store", e );
-        }
+        s3Client.createBucket( bucketName );
+        File uploadFile = new File( fileName );
+        PutObjectResult putObjectResult = s3Client.putObject( bucketName, uploadFile.getName(), uploadFile );
+        logger.info("Uploaded file etag={}", putObjectResult.getETag());
     }
 
 
@@ -386,7 +341,7 @@ public class WarehouseExport extends ExportingToolBase {
             OrganizationInfo info = managementService.getOrganizationByUuid( orgId );
 
             if ( info == null ) {
-                LOG.error( "Organization info is null!" );
+                logger.error( "Organization info is null!" );
                 System.exit( 1 );
             }
 
@@ -422,7 +377,7 @@ public class WarehouseExport extends ExportingToolBase {
 
     private void exportApplicationsForOrg( Entry<UUID, String> orgIdAndName, String queryString ) throws Exception {
 
-        LOG.info( "organization: {} / {}", orgIdAndName.getValue(), orgIdAndName.getKey() );
+        logger.info( "organization: {} / {}", orgIdAndName.getValue(), orgIdAndName.getKey() );
 
         String orgName = orgIdAndName.getValue();
 
@@ -432,7 +387,7 @@ public class WarehouseExport extends ExportingToolBase {
             String appName = appIdAndName.getValue();
             appName = appName.substring( appName.indexOf( '/' ) + 1 );
 
-            LOG.info( "application {} / {}", appName, appIdAndName.getKey() );
+            logger.info( "application {} / {}", appName, appIdAndName.getKey() );
 
             EntityManager em = emf.getEntityManager( appIdAndName.getKey() );
             Map<String, String[]> cfm = getCollectionFieldMap();
@@ -453,7 +408,7 @@ public class WarehouseExport extends ExportingToolBase {
 
                 Query query = Query.fromQL( queryString );
                 query.setLimit( MAX_ENTITY_FETCH );
-                query.setResultsLevel( Level.REFS );
+                query.setResultsLevel( Query.Level.REFS );
                 Results results = em.searchCollection( em.getApplicationRef(), collectionName, query );
 
                 while ( results.size() > 0 ) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseUpsert.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseUpsert.java b/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseUpsert.java
index 158abd6..901fb7e 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseUpsert.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseUpsert.java
@@ -31,7 +31,7 @@ import org.apache.commons.io.IOUtils;
 /** Upserts data from files found in an S3 bucket. */
 public class WarehouseUpsert extends ExportingToolBase {
 
-    private static final Logger LOG = LoggerFactory.getLogger( WarehouseUpsert.class );
+    private static final Logger logger = LoggerFactory.getLogger( WarehouseUpsert.class );
 
     public static final String DBHOST_PROPNAME = "usergrid.warehouse-export-dbhost";
     public static final String DBPORT_PROPNAME = "usergrid.warehouse-export-dbport";
@@ -84,14 +84,14 @@ public class WarehouseUpsert extends ExportingToolBase {
         String mainTableName = ( String ) properties.get( MAIN_TABLE_PROPNAME );
         try {
             con.createStatement().execute( createWarehouseTable( mainTableName ) );
-            LOG.info( "Created main table " + mainTableName );
+            logger.info( "Created main table " + mainTableName );
         }
         catch ( SQLException ex ) {
             if ( !ex.getMessage().contains( "already exists" ) ) {
-                LOG.error( "Error creating main table: " + ex.getMessage(), ex );
+                logger.error( "Error creating main table: " + ex.getMessage(), ex );
             }
             else {
-                LOG.info( "Using existing main table " + mainTableName );
+                logger.info( "Using existing main table " + mainTableName );
             }
         }
 
@@ -100,47 +100,47 @@ public class WarehouseUpsert extends ExportingToolBase {
         String dropStagingTable = String.format( "drop table %s", stagingTableName );
         try {
             con.createStatement().execute( dropStagingTable );
-            LOG.info( "Dropped existing staging table " + stagingTableName );
+            logger.info( "Dropped existing staging table " + stagingTableName );
         }
         catch ( SQLException ex ) {
             if ( !ex.getMessage().contains( "does not exist" ) ) {
-                LOG.error( "Error dropping staging table: " + ex.getMessage(), ex );
+                logger.error( "Error dropping staging table: " + ex.getMessage(), ex );
             }
             else {
-                LOG.info( "Using existing staging table " + stagingTableName );
+                logger.info( "Using existing staging table " + stagingTableName );
             }
         }
 
         // create staging table
-        LOG.info( "Creating new staging table" );
+        logger.info( "Creating new staging table" );
         con.createStatement().execute( createWarehouseTable( stagingTableName ) );
 
         // copy data from S3 into staging table
-        LOG.info( "Copying data from S3" );
+        logger.info( "Copying data from S3" );
         String copyFromS3 = String.format( "COPY %s FROM 's3://%s' "
                 + "CREDENTIALS 'aws_access_key_id=%s;aws_secret_access_key=%s' IGNOREHEADER 2 EMPTYASNULL",
                 stagingTableName, bucketName, accessId, secretKey );
-        LOG.debug( copyFromS3 );
+        logger.debug( copyFromS3 );
         con.createStatement().execute( copyFromS3 );
 
         // run update portion of upsert process
-        LOG.info( "Upsert: updating" );
+        logger.info( "Upsert: updating" );
         String upsertUpdate =
                 String.format( "UPDATE %s SET id = s.id FROM %s s WHERE %s.created = s.created ", mainTableName,
                         stagingTableName, mainTableName );
-        LOG.debug( upsertUpdate );
+        logger.debug( upsertUpdate );
         con.createStatement().execute( upsertUpdate );
 
         // insert new values in staging table into main table
-        LOG.info( "Upsert: inserting" );
+        logger.info( "Upsert: inserting" );
         String upsertInsert =
                 String.format( "INSERT INTO %s SELECT s.* FROM %s s LEFT JOIN %s n ON s.id = n.id WHERE n.id IS NULL",
                         mainTableName, stagingTableName, mainTableName );
-        LOG.debug( upsertInsert );
+        logger.debug( upsertInsert );
         con.createStatement().execute( upsertInsert );
 
         // drop staging table
-        LOG.info( "Dropping existing staging table" );
+        logger.info( "Dropping existing staging table" );
         con.createStatement().execute( dropStagingTable );
 
         // done!

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java b/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java
index b0f4f6e..49b110b 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java
@@ -17,19 +17,14 @@
 package org.apache.usergrid.tools.bean;
 
 
+import com.google.common.base.Preconditions;
+import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.index.query.CounterResolution;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
 
-import org.apache.usergrid.persistence.AggregateCounter;
-import org.apache.usergrid.persistence.AggregateCounterSet;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.Results;
-
-import com.google.common.base.Preconditions;
-import org.apache.usergrid.persistence.index.query.CounterResolution;
-
 
 /** @author zznate */
 public class MetricQuery {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
index 446aa91..2946ddb 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
@@ -17,27 +17,12 @@
 package org.apache.usergrid.tools;
 
 import org.apache.commons.lang.RandomStringUtils;
-import org.apache.usergrid.ServiceITSetup;
-import org.apache.usergrid.ServiceITSetupImpl;
-import org.apache.usergrid.ServiceITSuite;
-import org.apache.usergrid.management.ApplicationInfo;
-import org.apache.usergrid.management.OrganizationOwnerInfo;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.junit.ClassRule;
+import org.apache.usergrid.services.AbstractServiceIT;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import rx.Scheduler;
-import rx.schedulers.Schedulers;
 
 import java.io.File;
 import java.io.FileFilter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -46,47 +31,45 @@ import static org.junit.Assert.assertTrue;
 /**
  * TODO: better test, this is really just a smoke test.
  */
-public class ExportAppTest {
+public class ExportAppTest extends AbstractServiceIT {
     static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class );
-    
+
     int NUM_COLLECTIONS = 10;
-    int NUM_ENTITIES = 50; 
+    int NUM_ENTITIES = 50;
     int NUM_CONNECTIONS = 3;
 
-    @ClassRule
-    public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
 
     @org.junit.Test
     public void testBasicOperation() throws Exception {
-       
+
         String rand = RandomStringUtils.randomAlphanumeric( 10 );
-        
+
         // create app with some data
 
         String orgName = "org_" + rand;
         String appName = "app_" + rand;
-        
+
         ExportDataCreator creator = new ExportDataCreator();
         creator.startTool( new String[] {
                 "-organization", orgName,
                 "-application", appName,
-                "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort()
+            "-host", "localhost:" + setup.getCassSvc().getCassandraHostConfigurator().getPort()
         }, false);
-        
+
         long start = System.currentTimeMillis();
-        
+
         String directoryName = "target/export" + rand;
 
         ExportApp exportApp = new ExportApp();
         exportApp.startTool( new String[]{
                 "-application", orgName + "/" + appName,
                 "-writeThreads", "100",
-                "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(),
+                "-host", "localhost:" + setup.getCassSvc().getCassandraHostConfigurator().getPort(),
                 "-outputDir", directoryName
         }, false );
 
         logger.info( "100 read and 100 write threads = " + (System.currentTimeMillis() - start) / 1000 + "s" );
-        
+
         File exportDir = new File(directoryName);
         assertTrue( getFileCount( exportDir, "entities"    ) > 0 );
         assertTrue( getFileCount( exportDir, "connections" ) > 0 );
@@ -97,7 +80,7 @@ public class ExportAppTest {
         exportApp.startTool( new String[]{
                 "-application", orgName + "/" + appName,
                 "-writeThreads", "1",
-                "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(),
+                "-host", "localhost:" + setup.getCassSvc().getCassandraHostConfigurator().getPort(),
                 "-outputDir", directoryName + "1"
         }, false );
 
@@ -115,4 +98,4 @@ public class ExportAppTest {
             }
         } ).length;
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/e38ee3e9/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
index 9cce040..ceb6074 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
@@ -20,18 +20,16 @@ import com.google.common.collect.BiMap;
 import com.google.common.io.Files;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.RandomStringUtils;
-import org.apache.usergrid.ServiceITSetup;
-import org.apache.usergrid.ServiceITSetupImpl;
-import org.apache.usergrid.ServiceITSuite;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.management.OrganizationInfo;
 import org.apache.usergrid.management.OrganizationOwnerInfo;
 import org.apache.usergrid.management.UserInfo;
 import org.apache.usergrid.persistence.Entity;
 import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.services.AbstractServiceIT;
 import org.apache.usergrid.utils.UUIDUtils;
 import org.codehaus.jackson.JsonNode;
 import org.codehaus.jackson.map.ObjectMapper;
-import org.junit.ClassRule;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,18 +37,17 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.FilenameFilter;
-import java.util.*;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
 
 import static junit.framework.TestCase.assertNotNull;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
 import static org.junit.Assert.*;
 
 
-public class ExportImportAdminsTest {
+public class ExportImportAdminsTest extends AbstractServiceIT {
     static final Logger logger = LoggerFactory.getLogger( ExportImportAdminsTest.class );
-    
-    @ClassRule
-    public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
 
     @org.junit.Test
     public void testExportUserAndOrg() throws Exception {
@@ -81,7 +78,7 @@ public class ExportImportAdminsTest {
 
         ExportAdmins exportAdmins = new ExportAdmins();
         exportAdmins.startTool( new String[] {
-            "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(),
+            "-host", "localhost:" + setup.getCassSvc().getCassandraHostConfigurator().getPort(),
             "-outputDir", directoryName
         }, false );
 
@@ -148,7 +145,7 @@ public class ExportImportAdminsTest {
     public void testImportAdminUsersAndOrgs() throws Exception {
 
         // first: generate the data file with unique user and org IDs and names
-        
+
         // data contains three users each with a unique org, one user has a duplicate email
 
         String rand1 = RandomStringUtils.randomAlphanumeric( 10 );
@@ -214,8 +211,8 @@ public class ExportImportAdminsTest {
 
         ImportAdmins importAdmins = new ImportAdmins();
         importAdmins.startTool( new String[]{
-                "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(),
-                "-inputDir", tempDir.getAbsolutePath()
+            "-host", "localhost:" + setup.getCassSvc().getCassandraHostConfigurator().getPort(),
+            "-inputDir", tempDir.getAbsolutePath()
         }, false );
 
         // verify that users and orgs were created correctly
@@ -229,7 +226,7 @@ public class ExportImportAdminsTest {
         assertNotNull( "org 2 exists", orgInfo2 );
         List<UserInfo> org2_users = setup.getMgmtSvc().getAdminUsersForOrganization( org_uuid_2 );
         assertEquals( "org2 has two users", 2, org2_users.size() );
-        
+
         OrganizationInfo orgInfo3 = setup.getMgmtSvc().getOrganizationByUuid( org_uuid_3 );
         assertNotNull( "org 3 exists", orgInfo3 );
         List<UserInfo> org3_users = setup.getMgmtSvc().getAdminUsersForOrganization( org_uuid_3 );
@@ -237,7 +234,7 @@ public class ExportImportAdminsTest {
 
         BiMap<UUID, String> user1_orgs = setup.getMgmtSvc().getOrganizationsForAdminUser( user_uuid_1 );
         assertEquals( "user 1 has 2 orgs", 2, user1_orgs.size() );
-        
+
         BiMap<UUID, String> user2_orgs = setup.getMgmtSvc().getOrganizationsForAdminUser( user_uuid_2 );
         assertEquals( "user 2 has two orgs gained one from duplicate", 2, user2_orgs.size() );
 
@@ -248,10 +245,10 @@ public class ExportImportAdminsTest {
             logger.info("EXCEPTION EXPECTED");
         }
 
-        EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID );
+        EntityManager em = setup.getEmf().getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
         Entity user3 = em.get( user_uuid_3 );
         assertNull( "duplicate user does not exist", user3 );
 
 
     }
-}
\ No newline at end of file
+}


Mime
View raw message