usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject [04/50] [abbrv] usergrid git commit: Removing export and import files along with the other export that uses Jclouds
Date Mon, 02 May 2016 17:54:35 GMT
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cb1356c2/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
deleted file mode 100644
index 870b678..0000000
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
+++ /dev/null
@@ -1,1047 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.usergrid.management.export;
-
-
-import java.io.File;
-import java.io.FileReader;
-import java.util.*;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.util.concurrent.Service;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.usergrid.batch.service.JobSchedulerService;
-import org.apache.usergrid.utils.UUIDUtils;
-import org.jclouds.ContextBuilder;
-import org.jclouds.blobstore.BlobStore;
-import org.jclouds.blobstore.BlobStoreContext;
-import org.jclouds.blobstore.domain.Blob;
-import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
-import org.jclouds.logging.log4j.config.Log4JLoggingModule;
-import org.jclouds.netty.config.NettyPayloadModule;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.NewOrgAppAdminRule;
-import org.apache.usergrid.ServiceITSetup;
-import org.apache.usergrid.ServiceITSetupImpl;
-import org.apache.usergrid.batch.JobExecution;
-import org.apache.usergrid.cassandra.ClearShiroSubject;
-
-import org.apache.usergrid.management.ApplicationInfo;
-import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.management.UserInfo;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.SimpleEntityRef;
-import org.apache.usergrid.persistence.entities.JobData;
-import org.apache.usergrid.setup.ConcurrentProcessSingleton;
-
-import com.amazonaws.SDKGlobalConfiguration;
-import com.google.common.collect.ImmutableSet;
-import com.google.inject.Module;
-
-import static org.apache.usergrid.TestHelper.newUUIDString;
-import static org.apache.usergrid.TestHelper.uniqueApp;
-import static org.apache.usergrid.TestHelper.uniqueOrg;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-
-/**
- *
- *
- */
-public class ExportServiceIT {
-
-    private static final Logger logger = LoggerFactory.getLogger( ExportServiceIT.class );
-
-
-    @ClassRule
-    public static final ServiceITSetup setup = new ServiceITSetupImpl(  );
-
-    @Rule
-    public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
-
-    @Rule
-    public NewOrgAppAdminRule newOrgAppAdminRule = new NewOrgAppAdminRule( setup );
-
-    // app-level data generated only once
-    private UserInfo adminUser;
-    private OrganizationInfo organization;
-    private UUID applicationId;
-
-    private static String bucketPrefix;
-
-    private String bucketName;
-
-    @Before
-    public void setup() throws Exception {
-        logger.info("in setup");
-
-        // start the scheduler after we're all set up
-        try {
-
-            JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance().getSpringResource().getBean(JobSchedulerService.class);
-            if (jobScheduler.state() != Service.State.RUNNING) {
-                jobScheduler.startAsync();
-                jobScheduler.awaitRunning();
-            }
-        } catch ( Exception e ) {
-            logger.warn("Ignoring error starting jobScheduler, already started?", e);
-        }
-
-        adminUser = newOrgAppAdminRule.getAdminInfo();
-        organization = newOrgAppAdminRule.getOrganizationInfo();
-        applicationId = newOrgAppAdminRule.getApplicationInfo().getId();
-
-        setup.getEntityIndex().refresh(applicationId);
-    }
-
-
-    @Before
-    public void before() {
-
-        boolean configured =
-            !StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR))
-                && !StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR))
-                && !StringUtils.isEmpty(System.getProperty("bucketName"));
-
-        if ( !configured ) {
-            logger.warn("Skipping test because {}, {} and bucketName not " +
-                    "specified as system properties, e.g. in your Maven settings.xml file.",
-                new Object[] {
-                    SDKGlobalConfiguration.SECRET_KEY_ENV_VAR,
-                    SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR
-                });
-        }
-
-        Assume.assumeTrue( configured );
-
-        adminUser = newOrgAppAdminRule.getAdminInfo();
-        organization = newOrgAppAdminRule.getOrganizationInfo();
-        applicationId = newOrgAppAdminRule.getApplicationInfo().getId();
-
-        bucketPrefix = System.getProperty( "bucketName" );
-        bucketName = bucketPrefix + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
-    }
-
-
-    //Tests to make sure we can call the job with mock data and it runs.
-    // Connections won't save when run with maven, but on local builds it will.
-    @Ignore("Pending merge of export-feature branch")
-    public void testConnectionsOnCollectionExport() throws Exception {
-
-        File f = null;
-        int indexCon = 0;
-
-        try {
-            f = new File( "testFileConnections.json" );
-        }
-        catch ( Exception e ) {
-            // consumed because this checks to see if the file exists.
-            // If it doesn't then don't do anything and carry on.
-        }
-        f.deleteOnExit();
-
-        S3Export s3Export = new MockS3ExportImpl("testFileConnections.json" );
-
-        ExportService exportService = setup.getExportService();
-
-        String appName = newOrgAppAdminRule.getApplicationInfo().getName();
-        HashMap<String, Object> payload = payloadBuilder(appName);
-
-        payload.put( "organizationId", organization.getUuid() );
-        payload.put( "applicationId", applicationId );
-        payload.put( "collectionName", "users" );
-
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
-        //intialize user object to be posted
-        Map<String, Object> userProperties = null;
-        Entity[] entity;
-        entity = new Entity[2];
-        //creates entities
-        for ( int i = 0; i < 2; i++ ) {
-            userProperties = new LinkedHashMap<String, Object>();
-            userProperties.put( "username", "meatIsGreat" + i );
-            userProperties.put( "email", "grey" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
-
-            entity[i] = em.create( "users", userProperties );
-        }
-        //creates connections
-        em.createConnection( em.get( new SimpleEntityRef( "user", entity[0].getUuid() ) ), "Vibrations",
-            em.get( new SimpleEntityRef( "user", entity[1].getUuid() ) ) );
-        em.createConnection(
-                em.get( new SimpleEntityRef( "user", entity[1].getUuid()) ), "Vibrations",
-                em.get( new SimpleEntityRef( "user", entity[0].getUuid()) ) );
-
-        UUID exportUUID = exportService.schedule( payload );
-
-        TypeReference<HashMap<String,Object>> typeRef
-            = new TypeReference<HashMap<String,Object>>() {};
-
-        ObjectMapper mapper = new ObjectMapper();
-        HashMap<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
-
-        Map collectionsMap = (Map)jsonMap.get("collections");
-        List usersList = (List)collectionsMap.get("users");
-
-        int indexApp = 0;
-        for ( indexApp = 0; indexApp < usersList.size(); indexApp++ ) {
-            Map user = (Map)usersList.get( indexApp );
-            Map userProps = (Map)user.get("Metadata");
-            String uuid = ( String ) userProps.get( "uuid" );
-            if ( entity[0].getUuid().toString().equals( uuid ) ) {
-                break;
-            }
-        }
-
-        assertTrue("Uuid was not found in exported files. ", indexApp < usersList.size());
-
-        Map userMap = (Map)usersList.get( indexApp );
-        Map connectionsMap = (Map)userMap.get("connections");
-        assertNotNull( connectionsMap );
-
-        List vibrationsList = (List)connectionsMap.get( "Vibrations" );
-
-        assertNotNull( vibrationsList );
-
-        f.deleteOnExit();
-    }
-
-
-    @Test //Connections won't save when run with maven, but on local builds it will.
-    public void testConnectionsOnApplicationEndpoint() throws Exception {
-
-        File f = null;
-
-        try {
-            f = new File( "testConnectionsOnApplicationEndpoint.json" );
-        }
-        catch ( Exception e ) {
-            // consumed because this checks to see if the file exists.
-            // If it doesn't then don't do anything and carry on.
-        }
-
-        String fileName = "testConnectionsOnApplicationEndpoint.json";
-
-        S3Export s3Export = new MockS3ExportImpl( "testConnectionsOnApplicationEndpoint.json" );
-
-        ExportService exportService = setup.getExportService();
-
-        String appName = newOrgAppAdminRule.getApplicationInfo().getName();
-        HashMap<String, Object> payload = payloadBuilder( appName );
-
-        payload.put( "organizationId", organization.getUuid() );
-        payload.put( "applicationId", applicationId );
-
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
-
-        // intialize user object to be posted
-        Map<String, Object> userProperties = null;
-        Entity[] entity;
-        entity = new Entity[2];
-
-        // creates entities
-        for ( int i = 0; i < 2; i++ ) {
-            userProperties = new LinkedHashMap<String, Object>();
-            userProperties.put( "username", "billybob" + i );
-            userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
-
-            entity[i] = em.create( "users", userProperties );
-        }
-        setup.getEntityIndex().refresh(applicationId);
-        //creates connections
-        em.createConnection( em.get( new SimpleEntityRef( "user", entity[0].getUuid() ) ), "Vibrations",
-            em.get( new SimpleEntityRef( "user", entity[1].getUuid() ) ) );
-        em.createConnection(
-                em.get( new SimpleEntityRef( "user", entity[1].getUuid())), "Vibrations",
-                em.get( new SimpleEntityRef( "user", entity[0].getUuid())) );
-
-        UUID exportUUID = exportService.schedule( payload );
-
-        //create and initialize jobData returned in JobExecution.
-        JobData jobData = jobDataCreator(payload,exportUUID,s3Export);
-
-        JobExecution jobExecution = mock( JobExecution.class );
-        when( jobExecution.getJobData() ).thenReturn( jobData );
-
-        exportService.doExport( jobExecution );
-
-        TypeReference<HashMap<String,Object>> typeRef
-            = new TypeReference<HashMap<String,Object>>() {};
-
-        ObjectMapper mapper = new ObjectMapper();
-        HashMap<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
-
-        Map collectionsMap = (Map)jsonMap.get("collections");
-        List usersList = (List)collectionsMap.get("users");
-
-        int indexApp = 0;
-        for ( indexApp = 0; indexApp < usersList.size(); indexApp++ ) {
-            Map user = (Map)usersList.get( indexApp );
-            Map userProps = (Map)user.get("Metadata");
-            String uuid = ( String ) userProps.get( "uuid" );
-            if ( entity[0].getUuid().toString().equals( uuid ) ) {
-                break;
-            }
-        }
-
-        assertTrue("Uuid was not found in exported files. ", indexApp < usersList.size());
-
-        Map userMap = (Map)usersList.get( indexApp );
-        Map connectionsMap = (Map)userMap.get("connections");
-        assertNotNull( connectionsMap );
-
-        List vibrationsList = (List)connectionsMap.get( "Vibrations" );
-
-        assertNotNull( vibrationsList );
-
-        f.deleteOnExit();
-    }
-
-    @Test
-    public void testExportOneOrgCollectionEndpoint() throws Exception {
-
-        File f = null;
-
-
-        try {
-            f = new File( "exportOneOrg.json" );
-        }
-        catch ( Exception e ) {
-            //consumed because this checks to see if the file exists.
-            // If it doesn't then don't do anything and carry on.
-        }
-
-        //create another org to ensure we don't export it
-        newOrgAppAdminRule.createOwnerAndOrganization(
-            "noExport"+newUUIDString(),
-            "junkUserName"+newUUIDString(),
-            "junkRealName"+newUUIDString(),
-            newUUIDString()+"ugExport@usergrid.com",
-            "123456789" );
-
-        S3Export s3Export = new MockS3ExportImpl("exportOneOrg.json");
-      //  s3Export.setFilename( "exportOneOrg.json" );
-        ExportService exportService = setup.getExportService();
-
-        String appName = newOrgAppAdminRule.getApplicationInfo().getName();
-        HashMap<String, Object> payload = payloadBuilder(appName);
-
-        payload.put( "organizationId", organization.getUuid() );
-        payload.put( "applicationId", applicationId );
-        payload.put( "collectionName", "roles" );
-
-        UUID exportUUID = exportService.schedule( payload );
-
-        JobData jobData = jobDataCreator(payload,exportUUID,s3Export);
-
-
-        JobExecution jobExecution = mock( JobExecution.class );
-        when( jobExecution.getJobData() ).thenReturn( jobData );
-
-        exportService.doExport( jobExecution );
-
-        TypeReference<HashMap<String,Object>> typeRef
-            = new TypeReference<HashMap<String,Object>>() {};
-
-        ObjectMapper mapper = new ObjectMapper();
-        Map<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
-
-        Map collectionsMap = (Map)jsonMap.get("collections");
-        String collectionName = (String)collectionsMap.keySet().iterator().next();
-        List collection = (List)collectionsMap.get(collectionName);
-
-        for ( Object o : collection ) {
-            Map entityMap = (Map)o;
-            Map metadataMap = (Map)entityMap.get("Metadata");
-            String entityName = (String)metadataMap.get("name");
-            assertFalse( "junkRealName".equals( entityName ) );
-        }
-        f.deleteOnExit();
-    }
-
-
-    //
-    //creation of files doesn't always delete itself
-    @Test
-    public void testExportOneAppOnCollectionEndpoint() throws Exception {
-
-        final String orgName = uniqueOrg();
-        final String appName = uniqueApp();
-
-
-        File f = null;
-
-        try {
-            f = new File( "exportOneApp.json" );
-        }
-        catch ( Exception e ) {
-            // consumed because this checks to see if the file exists.
-            // If it doesn't, don't do anything and carry on.
-        }
-        f.deleteOnExit();
-
-
-        Entity appInfo = setup.getEmf().createApplicationV2(orgName, appName);
-        UUID applicationId = appInfo.getUuid();
-
-
-
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
-        //intialize user object to be posted
-        Map<String, Object> userProperties = null;
-        Entity[] entity;
-        entity = new Entity[1];
-        //creates entities
-        for ( int i = 0; i < 1; i++ ) {
-            userProperties = new LinkedHashMap<String, Object>();
-            userProperties.put( "username", "junkRealName" );
-            userProperties.put( "email", "test" + i + "@anuff.com" );
-            entity[i] = em.create( "user", userProperties );
-        }
-
-        S3Export s3Export = new MockS3ExportImpl("exportOneApp.json");
-        //s3Export.setFilename( "exportOneApp.json" );
-        ExportService exportService = setup.getExportService();
-
-        HashMap<String, Object> payload = payloadBuilder(appName);
-
-        payload.put( "organizationId", organization.getUuid() );
-        payload.put( "applicationId", applicationId );
-
-        UUID exportUUID = exportService.schedule( payload );
-
-        JobData jobData = jobDataCreator(payload,exportUUID,s3Export);
-
-        JobExecution jobExecution = mock( JobExecution.class );
-        when( jobExecution.getJobData() ).thenReturn( jobData );
-
-        exportService.doExport( jobExecution );
-
-        TypeReference<HashMap<String,Object>> typeRef
-            = new TypeReference<HashMap<String,Object>>() {};
-
-        ObjectMapper mapper = new ObjectMapper();
-        Map<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
-
-        Map collectionsMap = (Map)jsonMap.get("collections");
-        String collectionName = (String)collectionsMap.keySet().iterator().next();
-        List collection = (List)collectionsMap.get(collectionName);
-
-        for ( Object o : collection ) {
-            Map entityMap = (Map)o;
-            Map metadataMap = (Map)entityMap.get("Metadata");
-            String entityName = (String)metadataMap.get("name");
-            assertFalse( "junkRealName".equals( entityName ) );
-        }
-    }
-
-
-    @Test
-    public void testExportOneAppOnApplicationEndpointWQuery() throws Exception {
-
-        File f = null;
-        try {
-            f = new File( "exportOneAppWQuery.json" );
-        }
-        catch ( Exception e ) {
-            // consumed because this checks to see if the file exists.
-            // If it doesn't, don't do anything and carry on.
-        }
-        f.deleteOnExit();
-
-
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
-        //intialize user object to be posted
-        Map<String, Object> userProperties = null;
-        Entity[] entity;
-        entity = new Entity[1];
-        //creates entities
-        for ( int i = 0; i < 1; i++ ) {
-            userProperties = new LinkedHashMap<String, Object>();
-            userProperties.put( "name", "me" );
-            userProperties.put( "username", "junkRealName" );
-            userProperties.put( "email", "burp" + i + "@anuff.com" );
-            entity[i] = em.create( "users", userProperties );
-        }
-
-        S3Export s3Export = new MockS3ExportImpl("exportOneAppWQuery.json" );
-        ExportService exportService = setup.getExportService();
-
-        String appName = newOrgAppAdminRule.getApplicationInfo().getName();
-        HashMap<String, Object> payload = payloadBuilder(appName);
-
-        payload.put( "query", "select * where username = 'junkRealName'" );
-        payload.put( "organizationId", organization.getUuid() );
-        payload.put( "applicationId", applicationId );
-
-        UUID exportUUID = exportService.schedule( payload );
-
-        JobData jobData = jobDataCreator(payload,exportUUID,s3Export);
-
-        JobExecution jobExecution = mock( JobExecution.class );
-        when( jobExecution.getJobData() ).thenReturn( jobData );
-
-       setup.getEntityIndex().refresh(applicationId);
-
-        exportService.doExport( jobExecution );
-
-        TypeReference<HashMap<String,Object>> typeRef
-            = new TypeReference<HashMap<String,Object>>() {};
-
-        ObjectMapper mapper = new ObjectMapper();
-        Map<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
-
-        Map collectionsMap = (Map)jsonMap.get("collections");
-        String collectionName = (String)collectionsMap.keySet().iterator().next();
-        List collection = (List)collectionsMap.get( collectionName );
-
-        for ( Object o : collection ) {
-            Map entityMap = (Map)o;
-            Map metadataMap = (Map)entityMap.get("Metadata");
-            String entityName = (String)metadataMap.get("name");
-            assertFalse( "junkRealName".equals( entityName ) );
-        }
-    }
-
-
-    @Test
-    public void testExportOneCollection() throws Exception {
-
-        File f = null;
-        int entitiesToCreate = 5;
-
-        try {
-            f = new File( "exportOneCollection.json" );
-        }
-        catch ( Exception e ) {
-            // consumed because this checks to see if the file exists.
-            // If it doesn't, don't do anything and carry on.
-        }
-
-        f.deleteOnExit();
-
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
-
-        // em.createApplicationCollection( "qtsMagics" );
-        // intialize user object to be posted
-        Map<String, Object> userProperties = null;
-        Entity[] entity;
-        entity = new Entity[entitiesToCreate];
-        //creates entities
-        for ( int i = 0; i < entitiesToCreate; i++ ) {
-            userProperties = new LinkedHashMap<String, Object>();
-            userProperties.put( "username", "billybob" + i );
-            userProperties.put( "email", "test" + i + "@anuff.com" );
-            entity[i] = em.create( "qtsMagics", userProperties );
-        }
-
-        S3Export s3Export = new MockS3ExportImpl("exportOneCollection.json" );
-        ExportService exportService = setup.getExportService();
-
-        String appName = newOrgAppAdminRule.getApplicationInfo().getName();
-        HashMap<String, Object> payload = payloadBuilder(appName);
-
-        payload.put( "organizationId", organization.getUuid() );
-        payload.put( "applicationId", applicationId );
-        payload.put( "collectionName", "qtsMagics" );
-
-        UUID exportUUID = exportService.schedule( payload );
-
-        JobData jobData = jobDataCreator(payload,exportUUID,s3Export);
-
-        JobExecution jobExecution = mock( JobExecution.class );
-        when( jobExecution.getJobData() ).thenReturn( jobData );
-
-        setup.getEntityIndex().refresh(applicationId);
-
-        exportService.doExport( jobExecution );
-
-        TypeReference<HashMap<String,Object>> typeRef
-            = new TypeReference<HashMap<String,Object>>() {};
-
-        ObjectMapper mapper = new ObjectMapper();
-        HashMap<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
-
-        Map collectionsMap = (Map)jsonMap.get("collections");
-        String collectionName = (String)collectionsMap.keySet().iterator().next();
-        List collection = (List)collectionsMap.get( collectionName );
-
-        assertEquals(entitiesToCreate, collection.size());
-    }
-
-
-    @Test
-    public void testExportOneCollectionWQuery() throws Exception {
-
-        File f = null;
-        int entitiesToCreate = 5;
-
-        try {
-            f = new File( "exportOneCollectionWQuery.json" );
-        }
-        catch ( Exception e ) {
-            // consumed because this checks to see if the file exists.
-            // If it doesn't, don't do anything and carry on.
-        }
-        f.deleteOnExit();
-
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
-        em.createApplicationCollection( "baconators" );
-        setup.getEntityIndex().refresh(applicationId);
-
-        //initialize user object to be posted
-        Map<String, Object> userProperties = null;
-        Entity[] entity;
-        entity = new Entity[entitiesToCreate];
-
-        // creates entities
-        for ( int i = 0; i < entitiesToCreate; i++ ) {
-            userProperties = new LinkedHashMap<String, Object>();
-            userProperties.put( "username", "billybob" + i );
-            userProperties.put( "email", "test" + i + "@anuff.com" );
-            entity[i] = em.create( "baconators", userProperties );
-        }
-
-        S3Export s3Export = new MockS3ExportImpl("exportOneCollectionWQuery.json");
-        ExportService exportService = setup.getExportService();
-
-        String appName = newOrgAppAdminRule.getApplicationInfo().getName();
-        HashMap<String, Object> payload = payloadBuilder(appName);
-
-        payload.put( "query", "select * where username contains 'billybob0'" );
-        payload.put( "organizationId", organization.getUuid() );
-        payload.put( "applicationId", applicationId );
-        payload.put( "collectionName", "baconators" );
-
-        UUID exportUUID = exportService.schedule( payload );
-
-        JobData jobData = jobDataCreator( payload, exportUUID, s3Export );
-
-        JobExecution jobExecution = mock( JobExecution.class );
-        when( jobExecution.getJobData() ).thenReturn( jobData );
-
-        setup.getEntityIndex().refresh(applicationId);
-
-        exportService.doExport( jobExecution );
-
-        TypeReference<HashMap<String,Object>> typeRef
-            = new TypeReference<HashMap<String,Object>>() {};
-
-        ObjectMapper mapper = new ObjectMapper();
-        Map<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
-
-        Map collectionsMap = (Map)jsonMap.get("collections");
-        String collectionName = (String)collectionsMap.keySet().iterator().next();
-        List collectionList = (List)collectionsMap.get( collectionName );
-
-        assertEquals(1, collectionList.size());
-    }
-
-
-    @Test
-    public void testExportDoJob() throws Exception {
-
-        String appName = newOrgAppAdminRule.getApplicationInfo().getName();
-        HashMap<String, Object> payload = payloadBuilder(appName);
-
-        payload.put( "organizationId", organization.getUuid() );
-        payload.put( "applicationId", applicationId );
-
-
-        JobData jobData = new JobData();
-        jobData.setProperty( "jobName", "exportJob" );
-
-        // this needs to be populated with properties of export info
-        jobData.setProperty( "exportInfo", payload );
-
-        JobExecution jobExecution = mock( JobExecution.class );
-
-        when( jobExecution.getJobData() ).thenReturn( jobData );
-        when( jobExecution.getJobId() ).thenReturn( UUID.randomUUID() );
-
-        ExportJob job = new ExportJob();
-        ExportService eS = mock( ExportService.class );
-        job.setExportService( eS );
-        try {
-            job.doJob( jobExecution );
-        }
-        catch ( Exception e ) {
-            logger.error("Error doing job", e);
-            assert ( false );
-        }
-        assert ( true );
-    }
-
-    //tests that with empty job data, the export still runs.
-    @Test
-    public void testExportEmptyJobData() throws Exception {
-
-        JobData jobData = new JobData();
-
-        JobExecution jobExecution = mock( JobExecution.class );
-
-        when( jobExecution.getJobData() ).thenReturn( jobData );
-        when( jobExecution.getJobId() ).thenReturn( UUID.randomUUID() );
-
-        ExportJob job = new ExportJob();
-        S3Export s3Export = mock( S3Export.class );
-        //setup.getExportService().setS3Export( s3Export );
-        job.setExportService( setup.getExportService() );
-        try {
-            job.doJob( jobExecution );
-        }
-        catch ( Exception e ) {
-            assert ( false );
-        }
-        assert ( true );
-    }
-
-
-    @Test
-    public void testNullJobExecution() {
-
-        JobData jobData = new JobData();
-
-        JobExecution jobExecution = mock( JobExecution.class );
-
-        when( jobExecution.getJobData() ).thenReturn( jobData );
-        when( jobExecution.getJobId() ).thenReturn( UUID.randomUUID() );
-
-        ExportJob job = new ExportJob();
-        S3Export s3Export = mock( S3Export.class );
-       // setup.getExportService().setS3Export( s3Export );
-        job.setExportService( setup.getExportService() );
-        try {
-            job.doJob( jobExecution );
-        }
-        catch ( Exception e ) {
-            assert ( false );
-        }
-        assert ( true );
-    }
-
-
-    @Test
-    @Ignore("Pending merge of export-feature branch")
-    public void testIntegration100EntitiesOn() throws Exception {
-
-        if (logger.isDebugEnabled()) {
-            logger.debug("testIntegration100EntitiesOn(): starting...");
-        }
-
-        ExportService exportService = setup.getExportService();
-
-        String appName = newOrgAppAdminRule.getApplicationInfo().getName();
-        HashMap<String, Object> payload = payloadBuilder(appName);
-
-        payload.put( "organizationId", organization.getUuid() );
-        payload.put( "applicationId", applicationId );
-
-        // create five applications each with collection of five entities
-
-        for ( int i = 0; i < 5; i++ ) {
-
-            ApplicationInfo appMade = setup.getMgmtSvc().createApplication( organization.getUuid(), "superapp" + i );
-            EntityManager appEm = setup.getEmf().getEntityManager( appMade.getId() );
-
-            String collName = "superappCol" + i;
-            appEm.createApplicationCollection(collName);
-
-            Map<String, Object> entityLevelProperties = null;
-            Entity[] entNotCopied;
-            entNotCopied = new Entity[5];
-
-            for ( int index = 0; index < 5; index++ ) {
-                entityLevelProperties = new LinkedHashMap<String, Object>();
-                entityLevelProperties.put( "username", "bobso" + index );
-                entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
-                entNotCopied[index] = appEm.create( collName, entityLevelProperties );
-            }
-        }
-
-        // export the organization containing those apps and collections
-
-        UUID exportUUID = exportService.schedule( payload );
-
-        int maxRetries = 100;
-        int retries = 0;
-        while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
-            Thread.sleep(100);
-        }
-
-        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
-        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
-        Properties overrides = new Properties();
-        overrides.setProperty( "s3" + ".identity", accessId );
-        overrides.setProperty( "s3" + ".credential", secretKey );
-
-        // test that we can find the file that were exported to S3
-
-        BlobStore blobStore = null;
-        try {
-
-            final Iterable<? extends Module> MODULES = ImmutableSet.of(
-                new JavaUrlHttpCommandExecutorServiceModule(),
-                new Log4JLoggingModule(),
-                new NettyPayloadModule());
-
-            BlobStoreContext context = ContextBuilder.newBuilder("s3")
-                .credentials(accessId, secretKey)
-                .modules(MODULES)
-                .overrides(overrides)
-                .buildView(BlobStoreContext.class);
-
-            String expectedFileName = ((ExportServiceImpl) exportService)
-                .prepareOutputFileName(organization.getName(), "applications");
-
-            blobStore = context.getBlobStore();
-            if (!blobStore.blobExists(bucketName, expectedFileName)) {
-                blobStore.deleteContainer(bucketName);
-                Assert.fail("Blob does not exist: " + expectedFileName);
-            }
-            Blob bo = blobStore.getBlob(bucketName, expectedFileName);
-
-            Long numOfFiles = blobStore.countBlobs(bucketName);
-            Long numWeWant = 1L;
-            blobStore.deleteContainer(bucketName);
-            assertEquals(numOfFiles, numWeWant);
-            assertNotNull(bo);
-
-        } finally {
-            blobStore.deleteContainer(bucketName);
-        }
-    }
-
-    @Test
-    @Ignore("Pending merge of export-feature branch")
-    public void testIntegration100EntitiesForAllApps() throws Exception {
-
-        S3Export s3Export = new S3ExportImpl();
-        ExportService exportService = setup.getExportService();
-
-        String appName = newOrgAppAdminRule.getApplicationInfo().getName();
-        HashMap<String, Object> payload = payloadBuilder(appName);
-
-        OrganizationInfo orgMade = null;
-        ApplicationInfo appMade = null;
-        for ( int i = 0; i < 5; i++ ) {
-            orgMade = setup.getMgmtSvc().createOrganization( "minorboss" + i, adminUser, true );
-            for ( int j = 0; j < 5; j++ ) {
-                appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + j );
-
-                EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
-                customMaker.createApplicationCollection( "superappCol" + j );
-                //intialize user object to be posted
-                Map<String, Object> entityLevelProperties = null;
-                Entity[] entNotCopied;
-                entNotCopied = new Entity[1];
-                //creates entities
-                for ( int index = 0; index < 1; index++ ) {
-                    entityLevelProperties = new LinkedHashMap<String, Object>();
-                    entityLevelProperties.put( "derp", "bacon" );
-                    entNotCopied[index] = customMaker.create( "superappCol" + j, entityLevelProperties );
-                }
-            }
-        }
-
-        payload.put( "organizationId", orgMade.getUuid() );
-
-        UUID exportUUID = exportService.schedule( payload );
-        assertNotNull( exportUUID );
-
-        Thread.sleep( 3000 );
-
-        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
-        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
-
-        Properties overrides = new Properties();
-        overrides.setProperty( "s3" + ".identity", accessId );
-        overrides.setProperty( "s3" + ".credential", secretKey );
-
-        BlobStore blobStore = null;
-
-        try {
-            final Iterable<? extends Module> MODULES = ImmutableSet.of(
-                new JavaUrlHttpCommandExecutorServiceModule(),
-                new Log4JLoggingModule(),
-                new NettyPayloadModule() );
-
-            BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
-                .credentials(accessId, secretKey )
-                .modules(MODULES )
-                .overrides(overrides )
-                .buildView(BlobStoreContext.class );
-
-            blobStore = context.getBlobStore();
-
-            //Grab Number of files
-            Long numOfFiles = blobStore.countBlobs( bucketName );
-
-            String expectedFileName = ((ExportServiceImpl)exportService)
-                .prepareOutputFileName(organization.getName(), "applications");
-
-            //delete container containing said files
-            Blob bo = blobStore.getBlob(bucketName, expectedFileName);
-            Long numWeWant = 5L;
-            blobStore.deleteContainer( bucketName );
-
-            //asserts that the correct number of files was transferred over
-            assertEquals( numWeWant, numOfFiles );
-
-        }
-        finally {
-            blobStore.deleteContainer( bucketName );
-        }
-    }
-
-
-    @Test
-    @Ignore("Pending merge of export-feature branch")
-    public void testIntegration100EntitiesOnOneOrg() throws Exception {
-
-        S3Export s3Export = new S3ExportImpl();
-        ExportService exportService = setup.getExportService();
-
-        String appName = newOrgAppAdminRule.getApplicationInfo().getName();
-        HashMap<String, Object> payload = payloadBuilder(appName);
-
-        payload.put( "organizationId", organization.getUuid() );
-        payload.put( "applicationId", applicationId );
-
-        OrganizationInfo orgMade = null;
-        ApplicationInfo appMade = null;
-        for ( int i = 0; i < 100; i++ ) {
-            orgMade = setup.getMgmtSvc().createOrganization( "largerboss" + i, adminUser, true );
-            appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + i );
-
-            EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
-            customMaker.createApplicationCollection( "superappCol" + i );
-            //intialize user object to be posted
-            Map<String, Object> entityLevelProperties = null;
-            Entity[] entNotCopied;
-            entNotCopied = new Entity[20];
-            //creates entities
-            for ( int index = 0; index < 20; index++ ) {
-                entityLevelProperties = new LinkedHashMap<String, Object>();
-                entityLevelProperties.put( "username", "bobso" + index );
-                entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
-                entNotCopied[index] = customMaker.create( "superappCol", entityLevelProperties );
-            }
-        }
-
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
-
-        //intialize user object to be posted
-        Map<String, Object> userProperties = null;
-        Entity[] entity;
-        entity = new Entity[100];
-
-        //creates entities
-        for ( int i = 0; i < 100; i++ ) {
-            userProperties = new LinkedHashMap<String, Object>();
-            userProperties.put( "username", "bido" + i );
-            userProperties.put( "email", "bido" + i + "@anuff.com" );
-
-            entity[i] = em.create( "user", userProperties );
-        }
-
-        UUID exportUUID = exportService.schedule( payload );
-
-        while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) ) {}
-
-        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
-        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
-
-        Properties overrides = new Properties();
-        overrides.setProperty( "s3" + ".identity", accessId );
-        overrides.setProperty( "s3" + ".credential", secretKey );
-
-        Blob bo = null;
-        BlobStore blobStore = null;
-
-        try {
-            final Iterable<? extends Module> MODULES = ImmutableSet.of( new JavaUrlHttpCommandExecutorServiceModule(),
-                new Log4JLoggingModule(), new NettyPayloadModule() );
-
-            BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
-                .credentials( accessId, secretKey )
-                .modules( MODULES )
-                .overrides( overrides )
-                .buildView( BlobStoreContext.class );
-
-            String expectedFileName = ((ExportServiceImpl)exportService)
-                .prepareOutputFileName(organization.getName(), "applications");
-
-            blobStore = context.getBlobStore();
-            if ( !blobStore.blobExists( bucketName, expectedFileName ) ) {
-                assert ( false );
-            }
-            Long numOfFiles = blobStore.countBlobs( bucketName );
-            Long numWeWant = Long.valueOf( 1 );
-            assertEquals( numOfFiles, numWeWant );
-
-            bo = blobStore.getBlob( bucketName, expectedFileName );
-        }
-        catch ( Exception e ) {
-            assert ( false );
-        }
-
-        assertNotNull( bo );
-        blobStore.deleteContainer( bucketName );
-    }
-
-    public JobData jobDataCreator(HashMap<String, Object> payload,UUID exportUUID, S3Export s3Export) {
-        JobData jobData = new JobData();
-
-        jobData.setProperty( "jobName", "exportJob" );
-        jobData.setProperty( "exportInfo", payload );
-        jobData.setProperty( "exportId", exportUUID );
-        jobData.setProperty( "s3Export", s3Export );
-
-        return jobData;
-    }
-
-    /*Creates fake payload for testing purposes.*/
-    public HashMap<String, Object> payloadBuilder( String orgOrAppName ) {
-        HashMap<String, Object> payload = new HashMap<String, Object>();
-        Map<String, Object> properties = new HashMap<String, Object>();
-        Map<String, Object> storage_info = new HashMap<String, Object>();
-        storage_info.put( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR,
-            System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ) );
-        storage_info.put( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR,
-            System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ) );
-        storage_info.put( "bucket_location",  bucketName );
-
-        properties.put( "storage_provider", "s3" );
-        properties.put( "storage_info", storage_info );
-
-        payload.put( "path", orgOrAppName );
-        payload.put( "properties", properties );
-        return payload;
-    }
-}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/cb1356c2/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java
deleted file mode 100644
index bac08a2..0000000
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.management.export;
-
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.commons.io.FileUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * Streams/reads the information written from the export service to a file named "test.json"
- */
-public class MockS3ExportImpl implements S3Export {
-    private static final Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
-
-    private final String filename;
-
-
-    public MockS3ExportImpl (String filename) {
-        this.filename = filename;
-    }
-
-
-    @Override
-    public void copyToS3( File ephemeral, final Map<String,Object> exportInfo, String ignoredFileName ) {
-
-        File verifiedData = new File( filename );
-        try {
-            FileUtils.copyFile(ephemeral, verifiedData);
-            logger.info( "Copied file {} to {}", ephemeral.getAbsolutePath(), verifiedData );
-        }
-        catch ( IOException e ) {
-            logger.error("Error copying to S3", e);
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/cb1356c2/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportTrackerTest.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportTrackerTest.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportTrackerTest.java
deleted file mode 100644
index fab62ca..0000000
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportTrackerTest.java
+++ /dev/null
@@ -1,399 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.management.importer;
-
-
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.entities.FailedImport;
-import org.apache.usergrid.persistence.entities.FailedImportConnection;
-import org.apache.usergrid.persistence.entities.FailedImportEntity;
-import org.apache.usergrid.persistence.entities.FileImport;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Matchers.same;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-
-public class FileImportTrackerTest {
-
-    @Test
-    public void testSuccess() throws Exception {
-
-        final EntityManagerFactory emf = mock( EntityManagerFactory.class );
-        final EntityManager em = mock( EntityManager.class );
-        when( emf.getEntityManager( emf.getManagementAppId() ) ).thenReturn( em );
-
-        final UUID importFileId = UUIDGenerator.newTimeUUID();
-
-        final FileImport fileImport = new FileImport();
-        fileImport.setUuid( importFileId );
-
-        when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
-
-        final FileImportTracker fileImportTracker = new FileImportTracker( emf, fileImport, 1000 );
-
-        final long expectedCount = 100;
-
-        for ( long i = 0; i < expectedCount; i++ ) {
-            fileImportTracker.entityWritten();
-        }
-
-
-        fileImportTracker.complete();
-
-
-        ArgumentCaptor<FileImport> savedFileImport = ArgumentCaptor.forClass( FileImport.class );
-
-        verify( em ).update( savedFileImport.capture() );
-
-        final FileImport updated = savedFileImport.getValue();
-
-        assertSame( "Same instance should be updated", fileImport, updated );
-
-
-        assertEquals( "Same count expected", expectedCount, updated.getImportedEntityCount() );
-
-        assertNull( updated.getErrorMessage() );
-    }
-
-
-    @Test
-    public void testBoth() throws Exception {
-
-        // create mock em and emf
-
-        final EntityManagerFactory emf = mock( EntityManagerFactory.class );
-        final EntityManager em = mock( EntityManager.class );
-        when( emf.getEntityManager( emf.getManagementAppId() ) ).thenReturn( em );
-
-        // create tracker and call its entityWritten() and entityFailed() methods some number of times
-
-        final long expectedSuccess = 100;
-        final int expectedFails = 10;
-        final FileImport fileImport = new FileImport();
-        final UUID importFileId = UUIDGenerator.newTimeUUID();
-        fileImport.setUuid(importFileId);
-
-        // when tracker tries to create a FailedImportEntity, give it a mock
-        when(em.create(any(FailedImportEntity.class))).thenAnswer(new Answer<FailedImportEntity>() {
-            @Override
-            public FailedImportEntity answer(final InvocationOnMock invocation) throws Throwable {
-                return (FailedImportEntity) invocation.getArguments()[0];
-            }
-        });
-
-        final FileImportTracker fileImportTracker = new FileImportTracker(emf, fileImport, 1000);
-        for (long i = 0; i < expectedSuccess; i++) {
-            fileImportTracker.entityWritten();
-        }
-        for (int i = 0; i < expectedFails; i++) {
-            fileImportTracker.entityFailed("Failed to write entity " + i);
-        }
-        fileImportTracker.complete();
-
-
-        ArgumentCaptor<FileImport> savedFileImport = ArgumentCaptor.forClass( FileImport.class );
-
-        verify( em ).update( savedFileImport.capture() );
-
-        final FileImport updated = savedFileImport.getValue();
-
-        assertSame( "Same instance should be updated", fileImport, updated );
-        assertEquals( "Same count expected", expectedSuccess, updated.getImportedEntityCount() );
-        assertEquals( "Same fail expected", expectedFails, updated.getFailedEntityCount() );
-
-        // TODO why is error message not being set here?
-//        assertEquals( "Correct error message",
-//            "Failed to import some data.  See the import counters and errors.",
-//            updated.getErrorMessage() );
-
-        // TODO get the connections from the file import
-
-        ArgumentCaptor<FailedImportEntity> failedEntities =
-            ArgumentCaptor.forClass( FailedImportEntity.class );
-
-        verify( em, times( expectedFails ) )
-            .createConnection( same( fileImport ), eq( "errors" ), failedEntities.capture() );
-
-        // now check all our arguments
-
-        final List<FailedImportEntity> args = failedEntities.getAllValues();
-        assertEquals( "Same number of error connections created", expectedFails, args.size() );
-
-
-        for ( int i = 0; i < expectedFails; i++ ) {
-            final FailedImportEntity failedImport = args.get( i );
-            assertEquals( "Same message expected",
-                "Failed to write entity " + i, failedImport.getErrorMessage() );
-        }
-    }
-
-
-    @Test
-    public void explicitFail() throws Exception {
-
-        final EntityManagerFactory emf = mock( EntityManagerFactory.class );
-        final EntityManager em = mock( EntityManager.class );
-        when( emf.getEntityManager( emf.getManagementAppId() ) ).thenReturn( em );
-
-        final UUID importFileId = UUIDGenerator.newTimeUUID();
-
-
-        final FileImport fileImport = new FileImport();
-        fileImport.setUuid( importFileId );
-
-        when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
-
-
-        final FileImportTracker fileImportTracker = new FileImportTracker( emf, fileImport, 1000 );
-
-        final long expectedCount = 100;
-
-        for ( long i = 0; i < expectedCount; i++ ) {
-            fileImportTracker.entityWritten();
-        }
-
-
-        fileImportTracker.fatal( "Something bad happened" );
-
-
-        ArgumentCaptor<FileImport> savedFileImport = ArgumentCaptor.forClass( FileImport.class );
-
-        verify( em ).update( savedFileImport.capture() );
-
-        final FileImport updated = savedFileImport.getValue();
-
-        assertSame( "Same instance should be updated", fileImport, updated );
-
-
-        assertEquals( "Same count expected", expectedCount, updated.getImportedEntityCount() );
-
-        assertEquals( "Fail count is 0", 0, updated.getFailedEntityCount() );
-
-        assertEquals( "Correct expected message", "Something bad happened", updated.getErrorMessage() );
-
-        assertEquals( "Expected failed state", FileImport.State.FAILED, updated.getState() );
-    }
-
-
-    @Test
-    public void testAutoFlushSuccess() throws Exception {
-
-        final EntityManagerFactory emf = mock( EntityManagerFactory.class );
-        final EntityManager em = mock( EntityManager.class );
-        when( emf.getEntityManager( emf.getManagementAppId() ) ).thenReturn( em );
-
-        final UUID importFileId = UUIDGenerator.newTimeUUID();
-
-
-        final FileImport fileImport = new FileImport();
-        fileImport.setUuid( importFileId );
-
-        when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
-
-        //mock up returning the FailedEntityImport instance after save is invoked.
-
-
-        when( em.create( any( FailedImportConnection.class ) ) ).thenAnswer( new Answer<Object>() {
-            @Override
-            public Object answer( final InvocationOnMock invocation ) throws Throwable {
-                return invocation.getArguments()[0];
-            }
-        } );
-
-        final int expectedSuccess = 100;
-        final int expectedFails = 100;
-        final int expectedConnectionSuccess = 100;
-        final int expectedConnectionFails = 100;
-
-        final int expectedFlushCount = 2;
-        final int flushSize =
-            ( expectedFails + expectedFails + expectedConnectionSuccess + expectedConnectionFails )
-            / expectedFlushCount;
-
-        //set this to 1/2, so that we get saved twice
-        final FileImportTracker fileImportTracker = new FileImportTracker( emf, fileImport, flushSize );
-
-
-        for ( long i = 0; i < expectedSuccess; i++ ) {
-            fileImportTracker.entityWritten();
-        }
-
-
-        for ( int i = 0; i < expectedFails; i++ ) {
-            fileImportTracker.entityFailed( "Failed to write entity " + i );
-        }
-
-
-        for ( long i = 0; i < expectedConnectionSuccess; i++ ) {
-            fileImportTracker.connectionWritten();
-        }
-
-
-        for ( int i = 0; i < expectedConnectionFails; i++ ) {
-            fileImportTracker.connectionFailed( "Failed to write connection " + i );
-        }
-
-
-        fileImportTracker.complete();
-
-
-        ArgumentCaptor<FileImport> savedFileImport = ArgumentCaptor.forClass( FileImport.class );
-
-        verify( em, times( expectedFlushCount + 1 ) ).update( savedFileImport.capture() );
-
-        final FileImport updated = savedFileImport.getAllValues().get( 2 );
-
-        assertSame( "Same instance should be updated", fileImport, updated );
-
-        assertEquals( "Same count expected", expectedSuccess, updated.getImportedEntityCount() );
-
-        assertEquals( "Same fail expected", expectedFails, updated.getFailedEntityCount() );
-
-        assertEquals( "Same connection count expected", expectedConnectionSuccess,
-            updated.getImportedConnectionCount() );
-
-        assertEquals( "Same connection error count expected", expectedConnectionFails,
-            updated.getFailedConnectionCount() );
-
-        assertTrue(updated.getErrorMessage().startsWith("Failed to import") );
-
-        //TODO get the connections from the file import
-
-        ArgumentCaptor<FailedImport> failedEntities = ArgumentCaptor.forClass( FailedImport.class );
-
-        verify( em, times( expectedFails + expectedConnectionFails ) )
-            .createConnection( same( fileImport ), eq( "errors" ), failedEntities.capture() );
-
-        //now check all our arguments
-
-        final List<FailedImport> args = failedEntities.getAllValues();
-
-        assertEquals( "Same number of error connections created",
-            expectedFails + expectedConnectionFails, args.size() );
-
-
-        for ( int i = 0; i < expectedFails; i++ ) {
-
-            final FailedImport failedImport = args.get( i );
-
-            assertEquals( "Same message expected",
-                "Failed to write entity " + i, failedImport.getErrorMessage() );
-        }
-
-        for ( int i = expectedFails; i < expectedConnectionFails; i++ ) {
-
-            final FailedImport failedImport = args.get( i );
-
-            assertEquals( "Same message expected",
-                "Failed to write connection " + i, failedImport.getErrorMessage() );
-        }
-    }
-
-
-    @Test
-    public void loadingExistingState() throws Exception {
-
-        final EntityManagerFactory emf = mock( EntityManagerFactory.class );
-        final EntityManager em = mock( EntityManager.class );
-        when( emf.getEntityManager( emf.getManagementAppId() ) ).thenReturn( em );
-
-        final UUID importFileId = UUIDGenerator.newTimeUUID();
-
-
-        final FileImport fileImport = new FileImport();
-        fileImport.setUuid( importFileId );
-        fileImport.setImportedEntityCount( 1 );
-        fileImport.setFailedEntityCount( 2 );
-        fileImport.setImportedConnectionCount( 3 );
-        fileImport.setFailedConnectionCount( 4 );
-
-        when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
-
-        //mock up returning the FailedEntityImport instance after save is invoked.
-
-        FileImportTracker statistics = new FileImportTracker( emf, fileImport, 100 );
-
-        assertEquals( 1, statistics.getEntitiesWritten() );
-        assertEquals( 2, statistics.getEntitiesFailed() );
-
-        assertEquals( 3, statistics.getTotalEntityCount() );
-
-        assertEquals( 3, statistics.getConnectionsWritten() );
-        assertEquals( 4, statistics.getConnectionsFailed() );
-
-        assertEquals( 7, statistics.getTotalConnectionCount() );
-    }
-
-
-    @Test
-    public void failFast() throws Exception {
-
-        final EntityManagerFactory emf = mock( EntityManagerFactory.class );
-        final EntityManager em = mock( EntityManager.class );
-        when( emf.getEntityManager( emf.getManagementAppId() ) ).thenReturn( em );
-
-        final UUID importFileId = UUIDGenerator.newTimeUUID();
-
-
-        final FileImport fileImport = new FileImport();
-
-        when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
-
-        //mock up returning the FailedEntityImport instance after save is invoked.
-
-        FileImportTracker statistics = new FileImportTracker( emf, fileImport, 100 );
-
-
-        assertFalse( statistics.shouldStopProcessingEntities() );
-
-        assertFalse( statistics.shouldStopProcessingConnections() );
-
-
-        statistics.entityFailed( "test fail" );
-
-        assertTrue("We shouldn't process after a failure", statistics.shouldStopProcessingEntities());
-
-        statistics.connectionFailed( "test fail" );
-
-        assertTrue( "We shouldn't process after a failure", statistics.shouldStopProcessingConnections() );
-    }
-}
-

http://git-wip-us.apache.org/repos/asf/usergrid/blob/cb1356c2/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
deleted file mode 100644
index 10b6d28..0000000
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ /dev/null
@@ -1,686 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.usergrid.management.importer;
-
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-
-import org.jclouds.ContextBuilder;
-import org.jclouds.blobstore.BlobStore;
-import org.jclouds.blobstore.BlobStoreContext;
-import org.jclouds.blobstore.ContainerNotFoundException;
-import org.jclouds.blobstore.domain.PageSet;
-import org.jclouds.blobstore.domain.StorageMetadata;
-import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
-import org.jclouds.logging.log4j.config.Log4JLoggingModule;
-import org.jclouds.netty.config.NettyPayloadModule;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang3.StringUtils;
-
-import org.apache.usergrid.NewOrgAppAdminRule;
-import org.apache.usergrid.ServiceITSetup;
-import org.apache.usergrid.ServiceITSetupImpl;
-import org.apache.usergrid.batch.service.JobSchedulerService;
-import org.apache.usergrid.cassandra.ClearShiroSubject;
-import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.management.UserInfo;
-import org.apache.usergrid.management.export.ExportService;
-import org.apache.usergrid.persistence.ConnectionRef;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.SimpleEntityRef;
-import org.apache.usergrid.persistence.entities.FileImport;
-import org.apache.usergrid.persistence.entities.Import;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.Query.Level;
-import org.apache.usergrid.setup.ConcurrentProcessSingleton;
-
-import com.amazonaws.SDKGlobalConfiguration;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.util.concurrent.Service;
-import com.google.inject.Module;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-
-public class ImportCollectionIT {
-
-    private static final Logger logger = LoggerFactory.getLogger(ImportCollectionIT.class);
-
-    // app-level data generated only once
-    private static UserInfo adminUser;
-    private static OrganizationInfo organization;
-    private static UUID applicationId;
-
-    private static String bucketPrefix;
-
-    private String bucketName;
-
-
-    @Rule
-    public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
-
-    @ClassRule
-    public static final ServiceITSetup setup = new ServiceITSetupImpl(  );
-
-    @Rule
-    public NewOrgAppAdminRule newOrgAppAdminRule = new NewOrgAppAdminRule( setup );
-
-
-    @BeforeClass
-    public static void setup() throws Exception {
-
-        bucketPrefix = System.getProperty( "bucketName" );
-
-        // start the scheduler after we're all set up
-        JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance()
-            .getSpringResource().getBean( JobSchedulerService.class );
-
-        if ( jobScheduler.state() != Service.State.RUNNING ) {
-            jobScheduler.startAsync();
-            jobScheduler.awaitRunning();
-        }
-    }
-
-
-    @AfterClass
-    public static void tearDown() {
-        if ( !StringUtils.isEmpty( bucketPrefix )) {
-            deleteBucketsWithPrefix();
-        }
-    }
-
-
-    @Before
-    public void before() {
-
-        boolean configured =
-                   !StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR))
-                && !StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR))
-                && !StringUtils.isEmpty(System.getProperty("bucketName"));
-
-        if ( !configured ) {
-            logger.warn("Skipping test because {}, {} and bucketName not " +
-                "specified as system properties, e.g. in your Maven settings.xml file.",
-                new Object[] {
-                    SDKGlobalConfiguration.SECRET_KEY_ENV_VAR,
-                    SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR
-                });
-        }
-
-        Assume.assumeTrue( configured );
-
-        adminUser = newOrgAppAdminRule.getAdminInfo();
-        organization = newOrgAppAdminRule.getOrganizationInfo();
-        applicationId = newOrgAppAdminRule.getApplicationInfo().getId();
-
-
-        bucketName = bucketPrefix + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
-    }
-
-
-    @After
-    public void after() throws Exception {
-//        if (listener != null) {
-//            listener.stop();
-//            listener = null;
-//        }
-    }
-
-
-    // test case to check if a collection file is imported correctly
-    @Test
-    public void testExportImportCollection() throws Exception {
-
-        // create a collection of "thing" entities in the first application, export to S3
-        try {
-
-            final UUID targetAppId = setup.getMgmtSvc().createApplication(
-                organization.getUuid(), "target" + RandomStringUtils.randomAlphanumeric(10)).getId();
-
-            final EntityManager emApp1 = setup.getEmf().getEntityManager( targetAppId );
-            Map<UUID, Entity> thingsMap = new HashMap<>();
-            List<Entity> things = new ArrayList<>();
-            createTestEntities(emApp1, thingsMap, things, "thing");
-
-            deleteBucket();
-            exportCollection( emApp1, "things" );
-
-            // create new second application, import the data from S3
-
-            final UUID appId2 = setup.getMgmtSvc().createApplication(
-                organization.getUuid(), "second" + RandomStringUtils.randomAlphanumeric(10)).getId();
-
-            final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2);
-            importCollections(emApp2);
-
-            // make sure that it worked
-
-            logger.debug("\n\nCheck connections\n");
-
-            List<Entity> importedThings = emApp2.getCollection(
-                appId2, "things", null, Level.ALL_PROPERTIES).getEntities();
-            assertTrue( !importedThings.isEmpty() );
-
-            // two things have connections
-
-            int conCount = 0;
-            for ( Entity e : importedThings ) {
-                Results r = emApp2.getTargetEntities(e, "related", null, Level.IDS);
-                List<ConnectionRef> connections = r.getConnections();
-                conCount += connections.size();
-            }
-            assertEquals( 2, conCount );
-
-            logger.debug("\n\nCheck dictionaries\n");
-
-            // first two items have things in dictionary
-
-            EntityRef entity0 = importedThings.get(0);
-            Map connected0 = emApp2.getDictionaryAsMap(entity0, "connected_types");
-            Map connecting0 = emApp2.getDictionaryAsMap(entity0, "connected_types");
-            Assert.assertEquals( 1, connected0.size() );
-            Assert.assertEquals( 1, connecting0.size() );
-
-            EntityRef entity1 = importedThings.get(1);
-            Map connected1 = emApp2.getDictionaryAsMap(entity1, "connected_types");
-            Map connecting1 = emApp2.getDictionaryAsMap(entity1, "connected_types");
-            Assert.assertEquals( 1, connected1.size() );
-            Assert.assertEquals( 1, connecting1.size() );
-
-            // the rest rest do not have connections
-
-            EntityRef entity2 = importedThings.get(2);
-            Map connected2 = emApp2.getDictionaryAsMap(entity2, "connected_types");
-            Map connecting2 = emApp2.getDictionaryAsMap(entity2, "connected_types");
-            Assert.assertEquals( 0, connected2.size() );
-            Assert.assertEquals( 0, connecting2.size() );
-
-            // if entities are deleted from app1, they still exist in app2
-
-            logger.debug("\n\nCheck dictionary\n");
-            for ( Entity importedThing : importedThings ) {
-                emApp1.delete( importedThing );
-            }
-            setup.getEntityIndex().refresh(appId2);
-
-
-            importedThings = emApp2.getCollection(
-                appId2, "things", null, Level.ALL_PROPERTIES).getEntities();
-            assertTrue( !importedThings.isEmpty() );
-
-        } finally {
-            deleteBucket();
-        }
-    }
-
-
-    /**
-     * Test that an existing collection of entities can be updated
-     * by doing an import of entities identified by UUIDs.
-     */
-    @Test
-    public void testUpdateByImport() throws Exception {
-
-        // create collection of things in first application, export them to S3
-
-        final UUID targetAppId = setup.getMgmtSvc().createApplication(
-            organization.getUuid(), "target" + RandomStringUtils.randomAlphanumeric(10)).getId();
-
-        final EntityManager emApp1 = setup.getEmf().getEntityManager( targetAppId );
-
-        Map<UUID, Entity> thingsMap = new HashMap<>();
-        List<Entity> things = new ArrayList<>();
-        createTestEntities(emApp1, thingsMap, things, "thing");
-
-        deleteBucket();
-
-        try {
-            exportCollection(emApp1, "things");
-
-            // create new second application and import those things from S3
-
-            final UUID appId2 = setup.getMgmtSvc().createApplication(
-                organization.getUuid(), "second" + RandomStringUtils.randomAlphanumeric(10)).getId();
-
-            final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2);
-            importCollections(emApp2);
-
-
-            // update the things in the second application, export to S3
-
-            for (UUID uuid : thingsMap.keySet()) {
-                Entity entity = emApp2.get(uuid);
-                entity.setProperty("fuel_source", "Hydrogen");
-                emApp2.update(entity);
-            }
-
-            deleteBucket();
-            exportCollection(emApp2, "things");
-
-
-            // import the updated things back into the first application, check that they've been updated
-
-            importCollections(emApp1);
-
-            for (UUID uuid : thingsMap.keySet()) {
-                Entity entity = emApp1.get(uuid);
-                Assert.assertEquals("Hydrogen", entity.getProperty("fuel_source"));
-            }
-
-        } finally {
-            deleteBucket();
-        }
-    }
-
-
-   /**
-     * Simple import test but with multiple files.
-     */
-    @Test
-    public void testImportWithMultipleFiles() throws Exception {
-
-        deleteBucket();
-
-        try {
-
-            String targetAppName = "import-test-target-" + RandomStringUtils.randomAlphanumeric(10);
-            UUID targetAppId = setup.getMgmtSvc().createApplication(organization.getUuid(), targetAppName).getId();
-
-            // create 4 applications each with collection of 10 things, export all to S3
-            logger.debug("\n\nCreating 10 applications with 10 entities each\n");
-
-            for (int i = 0; i < 10; i++) {
-
-                String appName = "import-test-" + i + RandomStringUtils.randomAlphanumeric(10);
-                UUID appId = setup.getMgmtSvc().createApplication(organization.getUuid(), appName).getId();
-                EntityManager emApp = setup.getEmf().getEntityManager(appId);
-
-                Map<UUID, Entity> thingsMap = new HashMap<>();
-                List<Entity> things = new ArrayList<>();
-                createTestEntities(emApp, thingsMap, things, "thing");
-
-                exportCollection(emApp, "things");
-            }
-
-            // import all those exports from S3 into the default test application
-            logger.debug("\n\nImporting\n");
-
-            final EntityManager emDefaultApp = setup.getEmf().getEntityManager(targetAppId);
-            importCollections(emDefaultApp);
-
-            // we should now have 100 Entities in the default app
-
-            logger.debug("\n\nQuery to see if we now have 100 entities\n");
-
-            Query query = Query.fromQL("select *").withLimit(101);
-
-            List<Entity> importedThings = emDefaultApp.getCollection(
-                emDefaultApp.getApplicationId(), "things", query, Level.ALL_PROPERTIES).getEntities();
-
-            assertNotNull("importedThings must not be null", !importedThings.isEmpty());
-            assertTrue("importedThings must not be empty", !importedThings.isEmpty());
-            assertEquals("there must be 100 importedThings", 100, importedThings.size());
-
-        } finally {
-            deleteBucket();
-        }
-    }
-
-
-    /**
-     * TODO: Test that importing bad JSON will result in an informative error message.
-     */
-    @Test
-    public void testImportBadJson() throws Exception {
-
-        deleteBucket();
-
-        // export and upload a bad JSON file to the S3 bucket
-
-        String basePath = System.getProperty("target.directory")
-            + File.separator + "test-classes" + File.separator;
-
-        List<String> filenames = new ArrayList<>( 1 );
-        filenames.add( basePath + "testimport-bad-json.json");
-
-        S3Upload s3Upload = new S3Upload();
-        s3Upload.copyToS3(
-            System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR),
-            System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR),
-            bucketName, filenames );
-
-        // import bad JSON from from the S3 bucket
-
-        String appName = "import-test-" + RandomStringUtils.randomAlphanumeric(10);
-        UUID appId = setup.getMgmtSvc().createApplication(organization.getUuid(), appName).getId();
-
-        final EntityManager em = setup.getEmf().getEntityManager( appId );
-        UUID importId = importCollections(em);
-
-
-        // check that we got an informative error message back
-
-        List<Entity> importedThings = em.getCollection(
-            em.getApplicationId(), "things", null, Level.ALL_PROPERTIES).getEntities();
-
-        assertTrue("No entities should have been imported", importedThings.isEmpty());
-
-        ImportService importService = setup.getImportService();
-        Results results = importService.getFileImports( appId, importId, null, null );
-
-        assertEquals( "There is one", 1, results.size() );
-
-        assertEquals( "Entity is FileImport object",
-            FileImport.class, results.getEntity().getClass() );
-
-        FileImport fileImport = (FileImport)results.getEntity();
-
-        assertTrue( fileImport.getFileName().endsWith("testimport-bad-json.json"));
-
-        assertTrue( "Error message is correct",
-            fileImport.getErrorMessage().startsWith("Unexpected character ('<' (code 60))"));
-    }
-
-    @Test
-    public void testImportWithMultipleFilesSomeBad() throws Exception {
-
-        deleteBucket();
-
-        // upload good and badly formatted files to our S3 bucket
-
-        String basePath = System.getProperty("target.directory")
-            + File.separator + "test-classes" + File.separator;
-
-        List<String> filenames = new ArrayList<>( 3 );
-        filenames.add( basePath + "testimport-with-connections.json" );
-        filenames.add( basePath + "testimport-qtmagics.json" );
-        filenames.add( basePath + "testimport-bad-connection.json" );
-        filenames.add( basePath + "testimport-bad-json.json" );
-
-        S3Upload s3Upload = new S3Upload();
-        s3Upload.copyToS3(
-            System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR),
-            System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR),
-            bucketName, filenames );
-
-        // import all those files into the default test application
-
-        String targetAppName = "import-test-target-" + RandomStringUtils.randomAlphanumeric(10);
-        UUID targetAppId = setup.getMgmtSvc().createApplication(organization.getUuid(), targetAppName).getId();
-
-        final EntityManager emDefaultApp = setup.getEmf().getEntityManager( targetAppId );
-        UUID importId = importCollections(emDefaultApp);
-
-        {
-            List<Entity> importedThings = emDefaultApp.getCollection(
-                emDefaultApp.getApplicationId(), "connfails", null, Level.ALL_PROPERTIES).getEntities();
-            assertTrue( !importedThings.isEmpty());
-            assertEquals( 1, importedThings.size() );
-        }
-
-        {
-            List<Entity> importedThings = emDefaultApp.getCollection(
-                emDefaultApp.getApplicationId(), "qtmagics", null, Level.ALL_PROPERTIES).getEntities();
-            assertTrue(!importedThings.isEmpty());
-            assertEquals(5, importedThings.size());
-        }
-
-        {
-            List<Entity> importedThings = emDefaultApp.getCollection(
-                emDefaultApp.getApplicationId(), "badjsons", null, Level.ALL_PROPERTIES).getEntities();
-            assertTrue(!importedThings.isEmpty());
-            assertEquals( 4, importedThings.size() );
-        }
-
-        {
-            List<Entity> importedThings = emDefaultApp.getCollection(
-                emDefaultApp.getApplicationId(), "things", null, Level.ALL_PROPERTIES).getEntities();
-            assertTrue(!importedThings.isEmpty());
-            assertEquals( 10, importedThings.size() );
-        }
-
-        Thread.sleep(3000);
-
-        ImportService importService = setup.getImportService();
-        Results results = importService.getFileImports( targetAppId, importId, null, null );
-
-        assertEquals( "There four file imports", 4, results.size() );
-
-    }
-
-
-   //---------------------------------------------------------------------------------------------
-
-
-    /**
-     * Start import job that will import all collections in all data files in the S3 bucket.
-     */
-    private UUID importCollections(final EntityManager em) throws Exception {
-
-        logger.debug("\n\nImport into new app {}\n", em.getApplication().getName() );
-
-        final ImportService importService = setup.getImportService();
-
-        final Import importEntity = importService.schedule(em.getApplication().getUuid(),
-            new HashMap<String, Object>() {{
-            put( "path", organization.getName() + em.getApplication().getName() );
-            put( "organizationId", organization.getUuid() );
-            put( "applicationId", em.getApplication().getUuid() );
-            put( "properties", new HashMap<String, Object>() {{
-                put( "storage_provider", "s3" );
-                put( "storage_info", new HashMap<String, Object>() {{
-                    put( "s3_access_id",
-                        System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR) );
-                    put( "s3_key",
-                        System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ) );
-                    put( "bucket_location", bucketName );
-                }} );
-            }} );
-        }});
-
-        int maxRetries = 30;
-        int retries = 0;
-        Import.State state = importService.getState(importEntity.getUuid());
-        while (     !state.equals( Import.State.FINISHED )
-                 && !state.equals( Import.State.FAILED )
-                 && retries++ < maxRetries ) {
-
-            logger.debug("Waiting for import ({}) ...", state.toString());
-            Thread.sleep(1000);
-
-            state = importService.getState(importEntity.getUuid());
-        }
-
-        if ( retries >= maxRetries ) {
-            throw new RuntimeException("Max retries reached");
-        }
-        setup.getEntityIndex().refresh(em.getApplicationId());
-
-
-        return importEntity.getUuid();
-    }
-
-
-    /**
-     * Start export job that wilk export a specific collection to the S3 bucket.
-     */
-    private void exportCollection(
-        final EntityManager em, final String collectionName ) throws Exception {
-
-        logger.debug("\n\nExporting {} collection from application {}\n",
-            collectionName, em.getApplication().getName() );
-        setup.getEntityIndex().refresh(em.getApplicationId());
-
-
-        ExportService exportService = setup.getExportService();
-        UUID exportUUID = exportService.schedule( new HashMap<String, Object>() {{
-            put( "path", organization.getName() + em.getApplication().getName());
-            put( "organizationId",  organization.getUuid());
-            put( "applicationId", em.getApplication().getUuid() );
-            put( "collectionName", collectionName);
-            put( "properties", new HashMap<String, Object>() {{
-                 put( "storage_provider", "s3" );
-                 put( "storage_info", new HashMap<String, Object>() {{
-                     put( "s3_access_id",
-                         System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR) );
-                     put("s3_key",
-                         System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR));
-                    put( "bucket_location", bucketName );
-                }});
-            }});
-        }});
-
-        int maxRetries = 30;
-        int retries = 0;
-        while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
-            logger.debug("Waiting for export...");
-            Thread.sleep(1000);
-        }
-
-        if ( retries >= maxRetries ) {
-            throw new RuntimeException("Max retries reached");
-        }
-    }
-
-
-    /**
-     * Create test entities of a specified type.
-     * First two entities are connected.
-     */
-    private void createTestEntities(final EntityManager em,
-        Map<UUID, Entity> thingsMap, List<Entity> things, final String type) throws Exception {
-
-        logger.debug("\n\nCreating new {} collection in application {}\n",
-            type, em.getApplication().getName());
-
-        setup.getEntityIndex().refresh(em.getApplicationId());
-
-
-        List<Entity> created = new ArrayList<>();
-        for (int i = 0; i < 10; i++) {
-            final int count = i;
-            Entity e = em.create(type, new HashMap<String, Object>() {{
-                put("name", em.getApplication().getName() + "-" + type + "-" + count);
-                put("originalAppId", em.getApplication().getUuid());
-                put("originalAppName", em.getApplication().getName());
-            }});
-            thingsMap.put(e.getUuid(), e);
-            things.add(e);
-            created.add(e);
-        }
-
-        // first two things are related to each other
-        em.createConnection(new SimpleEntityRef(type, created.get(0).getUuid()),
-            "related", new SimpleEntityRef(type, created.get(1).getUuid()));
-        em.createConnection(new SimpleEntityRef(type, created.get(1).getUuid()),
-            "related", new SimpleEntityRef(type, created.get(0).getUuid()));
-
-        setup.getEntityIndex().refresh(em.getApplicationId());
-
-    }
-
-
-    /**
-     * Delete the configured s3 bucket.
-     */
-    public void deleteBucket() {
-
-        logger.debug("\n\nDelete bucket\n");
-
-        String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
-        String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);
-
-        Properties overrides = new Properties();
-        overrides.setProperty("s3" + ".identity", accessId);
-        overrides.setProperty("s3" + ".credential", secretKey);
-
-        final Iterable<? extends Module> MODULES = ImmutableSet
-            .of(new JavaUrlHttpCommandExecutorServiceModule(),
-                new Log4JLoggingModule(),
-                new NettyPayloadModule());
-
-        BlobStoreContext context =
-            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
-                .overrides(overrides).buildView(BlobStoreContext.class);
-
-        BlobStore blobStore = context.getBlobStore();
-        blobStore.deleteContainer( bucketName );
-    }
-
-
-    // might be handy if you need to clean up buckets
-    private static void deleteBucketsWithPrefix() {
-
-        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix );
-
-        String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
-        String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);
-
-        Properties overrides = new Properties();
-        overrides.setProperty("s3" + ".identity", accessId);
-        overrides.setProperty("s3" + ".credential", secretKey);
-
-        final Iterable<? extends Module> MODULES = ImmutableSet
-            .of(new JavaUrlHttpCommandExecutorServiceModule(),
-                new Log4JLoggingModule(),
-                new NettyPayloadModule());
-
-        BlobStoreContext context =
-            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
-                .overrides(overrides).buildView(BlobStoreContext.class);
-
-        BlobStore blobStore = context.getBlobStore();
-        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();
-
-        for ( Object o : blobStoreList.toArray() ) {
-            StorageMetadata s = (StorageMetadata)o;
-
-            if ( s.getName().startsWith( bucketPrefix )) {
-                try {
-                    blobStore.deleteContainer(s.getName());
-                } catch ( ContainerNotFoundException cnfe ) {
-                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe );
-                }
-                logger.debug("Deleted bucket {}", s.getName());
-            }
-        }
-    }
-}


Mime
View raw message