usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snoopd...@apache.org
Subject [22/50] [abbrv] git commit: Correctly writes export file to s3 using file stream buffers.
Date Tue, 25 Feb 2014 06:21:36 GMT
Correctly writes export file to s3 using file stream buffers.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/15db8f4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/15db8f4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/15db8f4c

Branch: refs/pull/60/head
Commit: 15db8f4cf5864ef0530071ed37883bac44f4a282
Parents: 68bd76b
Author: George Reyes <ApigeeCorporation@George-Reyess-MacBook-Pro.local>
Authored: Wed Feb 12 17:09:14 2014 -0800
Committer: George Reyes <ApigeeCorporation@George-Reyess-MacBook-Pro.local>
Committed: Wed Feb 12 17:09:14 2014 -0800

----------------------------------------------------------------------
 .../management/export/ExportServiceImpl.java    | 54 ++++++++++++--------
 .../usergrid/management/export/S3Export.java    |  4 +-
 .../management/export/S3ExportImpl.java         | 13 +++--
 3 files changed, 45 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/15db8f4c/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
index c40f7f1..099f23e 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
@@ -1,9 +1,12 @@
 package org.usergrid.management.export;
 
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.InputStream;
+import java.io.ObjectOutputStream;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -194,9 +197,14 @@ public class ExportServiceImpl implements ExportService{
         return jobUUID;
     }
 
-
+//TODO: make multipart streaming functional
+    //currently only stores the collection in memory then flushes it.
     private void exportApplicationsForOrg( Map.Entry<UUID, String> organization,final
ExportInfo config ) throws Exception {
 
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        ObjectOutputStream oos = new ObjectOutputStream(baos);
+
+
 
 
         logger.info( "" + organization );
@@ -219,7 +227,7 @@ public class ExportServiceImpl implements ExportService{
 
             String appFileName =  prepareOutputFileName( "application", application.getValue()
);
 
-            JsonGenerator jg = getJsonGenerator( new File( appFileName ) );
+            JsonGenerator jg = getJsonGenerator( oos );
 
             // load the dictionary
 
@@ -258,16 +266,17 @@ public class ExportServiceImpl implements ExportService{
             nsEntity.setMetadata( "collections", collections );
 
             jg.writeStartArray();
-            jg.writeObject( nsEntity );
+            //jg.writeObject( nsEntity );
 
             // Create a GENERATOR for the application collections.
             //JsonGenerator collectionsJg = getJsonGenerator( createOutputFile( "collections",
application.getValue() ) );
 
-            String collectionsFilename = prepareOutputFileName( "collections","appDummyName"
);
-            JsonGenerator collectionsJg = getJsonGenerator( new File( collectionsFilename
) );
+            //String collectionsFilename = prepareOutputFileName( "collections","appDummyName"
);
+            //JsonGenerator collectionsJg = getJsonGenerator( oos );
 
-            collectionsJg.writeStartObject();
 
+            //collectionsJg.writeStartObject();
+           // jg.writeStartObject();
             Map<String, Object> metadata = em.getApplicationCollectionMetadata();
             //don't need to echo as not a command line tool anymore
             //echo( JsonUtils.mapToFormattedJsonString( metadata ) );
@@ -287,10 +296,10 @@ public class ExportServiceImpl implements ExportService{
                     for ( Entity entity : entities ) {
                         // Export the entity first and later the collections for
                         // this entity.
-                        jg.writeObject( entity );
+                        //jg.writeObject( entity );
                         //echo( entity );
 
-                        saveCollectionMembers( collectionsJg, em, application.getValue(),
entity );
+                        saveCollectionMembers( jg, em, application.getValue(), entity );
                     }
 
                     //we're done
@@ -306,15 +315,20 @@ public class ExportServiceImpl implements ExportService{
             }
 
             // Close writer for the collections for this application.
-            collectionsJg.writeEndObject();
-            collectionsJg.close();
+           // collectionsJg.writeEndObject();
+           // collectionsJg.close();
 
             // Close writer and file for this application.
+
+           // logger.warn();
             jg.writeEndArray();
             jg.close();
-            s3Export.copyToS3( appFileName, config );
+            oos.flush();
+            oos.close();
+
+            InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+            s3Export.copyToS3( is, config );
             //below line doesn't copy very good data anyways.
-            //copyToS3( collectionsFilename, config );
         }
     }
 
@@ -335,7 +349,7 @@ public class ExportServiceImpl implements ExportService{
             return;
         }
 
-        jg.writeFieldName( entity.getUuid().toString() );
+       // jg.writeFieldName( entity.getUuid().toString() );
         jg.writeStartObject();
 
         for ( String collectionName : collections ) {
@@ -368,15 +382,15 @@ public class ExportServiceImpl implements ExportService{
         jg.writeEndObject();
     }
 
-    protected JsonGenerator getJsonGenerator( String outFile ) throws IOException {
-        return getJsonGenerator( new File( outputDir, outFile ) );
-    }
+   // protected JsonGenerator getJsonGenerator( String outFile ) throws IOException {
+   //     return getJsonGenerator( new File( outputDir, outFile ) );
+   // }
 
 
-    protected JsonGenerator getJsonGenerator( File outFile ) throws IOException {
+    protected JsonGenerator getJsonGenerator( ObjectOutputStream oos ) throws IOException
{
         //TODO:shouldn't the below be UTF-16?
-        PrintWriter out = new PrintWriter( outFile, "UTF-8" );
-        JsonGenerator jg = jsonFactory.createJsonGenerator( out );
+        //PrintWriter out = new PrintWriter( outFile, "UTF-8" );
+        JsonGenerator jg = jsonFactory.createJsonGenerator( oos );
         jg.setPrettyPrinter( new DefaultPrettyPrinter() );
         jg.setCodec( new ObjectMapper() );
         return jg;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/15db8f4c/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
index 6895da5..cc4486c 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
@@ -1,6 +1,8 @@
 package org.usergrid.management.export;
 
 
+import java.io.InputStream;
+
 import org.usergrid.management.ExportInfo;
 
 
@@ -9,6 +11,6 @@ import org.usergrid.management.ExportInfo;
  *
  */
 public interface S3Export {
-    void copyToS3( String fileName, ExportInfo exportInfo );
+    void copyToS3( InputStream inputStream, ExportInfo exportInfo );
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/15db8f4c/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
b/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
index fc8e281..52afb71 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
@@ -1,7 +1,7 @@
 package org.usergrid.management.export;
 
 
-import java.io.File;
+import java.io.InputStream;
 import java.util.Properties;
 
 import org.jclouds.ContextBuilder;
@@ -27,8 +27,9 @@ import com.google.inject.Module;
  *
  */
 public class S3ExportImpl implements S3Export {
+
     @Override
-    public void copyToS3( String fileName, final ExportInfo exportInfo ) {
+    public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo ) {
 
         Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
         /*won't need any of the properties as I have the export info*/
@@ -63,11 +64,12 @@ public class S3ExportImpl implements S3Export {
         }
 
         try {
-            File file = new File( fileName );
+
+
             AsyncBlobStore blobStore = context.getAsyncBlobStore();
             BlobBuilder blobBuilder =
-                    blobStore.blobBuilder( file.getName() ).payload( file ).calculateMD5().contentType(
"text/plain" )
-                             .contentLength( file.length() );
+                    blobStore.blobBuilder( "test.json" ).payload( inputStream ).calculateMD5().contentType(
"text/plain" );
+
 
             Blob blob = blobBuilder.build();
 
@@ -78,6 +80,7 @@ public class S3ExportImpl implements S3Export {
         catch ( Exception e ) {
             logger.error( "Error uploading to blob store", e );
         }
+
     }
 
 }


Mime
View raw message