usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [16/50] [abbrv] git commit: Added a batch job that will eventually need to be converted to a only one job instance. First pass doesn’t need multiple jobs Will also need to have the job launch the schedule and have it be a component then have THAT method
Date Tue, 25 Feb 2014 06:21:30 GMT
Added a batch job that will eventually need to be converted to a only one job instance. First
pass doesn’t need multiple jobs
Will also need to have the job launch the schedule and have it be a component then have THAT
method call the doExport method.
Added some logging to let you debug if there is no organization or no application. dummy data
is set for the UUID as there is no class to hold that information yet.
Added a comment to export info that dictates the next steps to get jobs working.
Added a new test that creates 100 entities, but the test doesn’t contain file checking yet.
Re included authorization, but will most likely be taken out. Also re added the sch command
but it will to call the doExport not just call both methods.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b45a6fdc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b45a6fdc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b45a6fdc

Branch: refs/pull/60/merge
Commit: b45a6fdce32d9fe54ef6702a59981ad3360d9cb6
Parents: 7a8ec96
Author: GERey <greyes@apigee.com>
Authored: Fri Feb 7 20:38:00 2014 -0800
Committer: GERey <greyes@apigee.com>
Committed: Fri Feb 7 20:38:00 2014 -0800

----------------------------------------------------------------------
 .../rest/management/ManagementResource.java     |  9 ++--
 .../rest/management/ManagementResourceIT.java   | 46 +++++++++++++++++++-
 .../org/usergrid/management/ExportInfo.java     | 20 ++++++++-
 .../management/export/ExportBatchJob.java       | 36 +++++++++++++++
 .../management/export/ExportServiceImpl.java    | 40 +++++++++++++++--
 5 files changed, 141 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b45a6fdc/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
b/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
index b2e6d36..4985239 100644
--- a/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/usergrid/rest/management/ManagementResource.java
@@ -450,18 +450,21 @@ public class ManagementResource extends AbstractContextResource {
     @POST
     @Path( "export" )
     @Consumes(MediaType.APPLICATION_JSON)
-    public Response exportPostJson (@Context UriInfo ui, Map<String, Object> json,
+    public Response exportPostJson (@Context UriInfo ui,@HeaderParam( "Authorization" ) String
authorization,
+                                    Map<String, Object> json,
                                     @QueryParam( "callback" ) @DefaultValue( "" ) String
callback){
 
 
         OAuthResponse response = null;
+        String client_id;
+        String client_secret;
+
         try {
 
         //parse the json into some useful object (the config params)
         ExportInfo objEx = new ExportInfo(json);
 
-
-        //exportService.schedule(objEx);
+        exportService.schedule(objEx);
         exportService.doExport( objEx );
         }
         catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b45a6fdc/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
b/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
index 0d6f8e0..6e54feb 100644
--- a/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/usergrid/rest/management/ManagementResourceIT.java
@@ -22,6 +22,7 @@ import java.util.Map;
 import javax.ws.rs.core.MediaType;
 
 import org.codehaus.jackson.JsonNode;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.usergrid.cassandra.Concurrent;
 import org.usergrid.management.OrganizationInfo;
@@ -527,8 +528,9 @@ public class ManagementResourceIT extends AbstractRestIT {
 
         //assertEquals( Status.OK, status );
     }
-
-    @Test
+//test needs to be updated to do validation of files/ and validation of file created in s3.
+    // eventually will need to simulate dropped connections as well.
+    @Ignore
     public void exportCallSuccessful() throws Exception {
         Status responseStatus = Status.OK;
         JsonNode node = null;
@@ -559,4 +561,44 @@ public class ManagementResourceIT extends AbstractRestIT {
 
         assertEquals( Status.OK, responseStatus );
     }
+
+    @Ignore
+    public void exportCallCreationEntities100() throws Exception {
+        Status responseStatus = Status.OK;
+        JsonNode node = null;
+
+        HashMap<String, Object> payload = new HashMap<String, Object>();
+        Map<String, Object> properties = new HashMap<String, Object>();
+        Map<String, Object> storage_info = new HashMap<String, Object>();
+        //TODO: make sure to put a valid admin token here.
+        storage_info.put( "admin_token", adminToken() );
+        //TODO: always put dummy values here and ignore this test.
+        storage_info.put( "s3_key","insert key here" );
+        storage_info.put( "s3_accessId","insert access id here");
+        storage_info.put( "bucket_location","insert bucket name here");
+
+        properties.put( "storage_provider","s3");
+        properties.put( "storage_info",storage_info);
+
+        payload.put( "path", "test-organization/test-app/user");
+        payload.put( "properties", properties);
+
+        for (int i = 0; i < 100; i++) {
+            Map<String, String> userCreation = hashMap( "type", "app_user" ).map( "name",
"fred"+i );
+
+            node = resource().path( "/test-organization/test-app/app_users" )
+                                  .queryParam( "access_token", access_token ).accept( MediaType.APPLICATION_JSON
)
+                                  .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class,
userCreation );
+        }
+
+        try {
+            node = resource().path( "/management/export" ).accept( MediaType.APPLICATION_JSON
)
+                             .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class,
payload );
+        }
+        catch ( UniformInterfaceException uie ) {
+            responseStatus = uie.getResponse().getClientResponseStatus();
+        }
+
+        assertEquals( Status.OK, responseStatus );
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b45a6fdc/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
index 5dbb771..1a272e4 100644
--- a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
+++ b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
@@ -6,7 +6,7 @@ import java.util.Map;
  * Created by ApigeeCorporation on 1/31/14.
  */
 //TODO: Documentation on this class.
-public class ExportInfo {
+public class ExportInfo { //extends Entity something {
 
     private String path;
     private Map<String, Object> properties;
@@ -17,6 +17,24 @@ public class ExportInfo {
     private String s3_key;
     private String bucket_location;
 
+
+    /**
+     *
+     *the sch system doesn't make any assumes about the job or how it works.
+     * and so if I need additional information to be persistant.
+     *
+     * The way to save data between queue and storing it.
+     *
+     * in my case, create a export entity. before I schedule the job and it'll have the pending
state in it and
+     * all the information I need to run. Then I'll pass the ID of the export info I saved
in a collection and i'll put that in the jbo
+     * data.
+     *
+     * persist the state in mechanisum that they can all access.
+     *
+     * I could make it a class and I can make it an entity. That way I can get it in and
out.
+     * doesn't get exposed to the user.
+     */
+
     public ExportInfo ( Map<String, Object> exportData) {
         path = (String) exportData.get("path");
         properties = (Map) exportData.get("properties");

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b45a6fdc/stack/services/src/main/java/org/usergrid/management/export/ExportBatchJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportBatchJob.java
b/stack/services/src/main/java/org/usergrid/management/export/ExportBatchJob.java
new file mode 100644
index 0000000..08f3701
--- /dev/null
+++ b/stack/services/src/main/java/org/usergrid/management/export/ExportBatchJob.java
@@ -0,0 +1,36 @@
+package org.usergrid.management.export;
+
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.usergrid.batch.Job;
+import org.usergrid.batch.JobExecution;
+import org.usergrid.persistence.entities.JobData;
+
+
+/**
+ *
+ *
+ */
+public class ExportBatchJob implements Job {
+
+    private static final Logger logger = LoggerFactory.getLogger( ExportBatchJob.class );
+
+    //TODO: ask if the service manager is needed here. Maybe in order to see if process.
+    @Autowired
+    protected ExportService exportService;
+
+
+    @Override
+    public void execute( final JobExecution execution ) throws Exception {
+
+        logger.info("execute ExportBatchJob {}", execution);
+
+        JobData jobData = execution.getJobData();
+        UUID jobUUID = (UUID) jobData.getProperty( "jobUUID" );
+
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b45a6fdc/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
index 5fd6a58..d34fb23 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
@@ -36,6 +36,7 @@ import org.usergrid.persistence.EntityManagerFactory;
 import org.usergrid.persistence.Query;
 import org.usergrid.persistence.Results;
 import org.usergrid.persistence.cassandra.CassandraService;
+import org.usergrid.persistence.entities.JobData;
 
 import com.google.common.collect.BiMap;
 import com.google.common.collect.ImmutableSet;
@@ -49,6 +50,8 @@ import com.google.inject.Module;
  */
 public class ExportServiceImpl implements ExportService{
 
+
+    private static final Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class
);
     //dependency injection
     //inject scheduler - needs to be auto wired
     private SchedulerService sch;
@@ -80,11 +83,39 @@ public class ExportServiceImpl implements ExportService{
 
     @Override
     public void schedule( final ExportInfo config ) {
-        //SchedulerServiceImpl sch =
 
-        //validate org exists
-        //validate user has access to or
+        //validate that org exists,then app, then collection.
+        String pathToBeParsed = config.getPath();
+        //split the path so that you can verify that the organization and the app exist.
+        String[] pathItems = pathToBeParsed.split( "/" );
+        try {
+            managementService.getOrganizationByName( pathItems[0] );
+        }
+        catch ( Exception e ) {
+            logger.error( "Organization doesn't exist" );
+        }
+
+        try {
+            managementService.getApplicationInfo( pathItems[1] );
+        }
+        catch ( Exception e ) {
+            logger.error( "Application doesn't exist" );
+        }
+
+
+        //TODO: parse path and make sure all the things you need actually exist. then throw
+        // good error messages when not found.
+
+        //  managementService.getOrganizationByName(  )
+        //validate user has access key to org (rather valid user has admin access token)
+            //this is token validation
         //schedule the job
+        JobData jobData = new JobData();
+
+        jobData.setProperty( "jobId", "0001" );//TODO: store uuid here, give export job uuid.
+        long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
+        sch.createJob( "queueExportJob",soonestPossible, jobData );
+
 
     }
 
@@ -123,6 +154,7 @@ public class ExportServiceImpl implements ExportService{
             if ( info == null ) {
 
                 //logger.error( "Organization info is null!" );
+                //TODO: remove all instances of system.exit in code case that was adapated.
                 System.exit( 1 );
             }
 
@@ -168,7 +200,7 @@ public class ExportServiceImpl implements ExportService{
 
     private void exportApplicationsForOrg( Map.Entry<UUID, String> organization,final
ExportInfo config ) throws Exception {
 
-        Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
+
 
         logger.info( "" + organization );
 


Mime
View raw message