usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject incubator-usergrid git commit: Added new file with an export that should only export admin users. Yet to be tested.
Date Fri, 10 Apr 2015 03:12:23 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/exportImportTool [created] a29d09eea


Added new file with an export that should only export admin users. Yet to be tested.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a29d09ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a29d09ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a29d09ee

Branch: refs/heads/exportImportTool
Commit: a29d09eea702d99ea30ed8931222fde34a80ed07
Parents: 9ef73e7
Author: GERey <greyes@apigee.com>
Authored: Thu Apr 9 20:12:21 2015 -0700
Committer: GERey <greyes@apigee.com>
Committed: Thu Apr 9 20:12:21 2015 -0700

----------------------------------------------------------------------
 .../org/apache/usergrid/tools/ExportAdmins.java | 453 +++++++++++++++++++
 1 file changed, 453 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a29d09ee/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
new file mode 100644
index 0000000..5f34855
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@ -0,0 +1,453 @@
+
+
+/**
+ * Created by ApigeeCorporation on 4/9/15.
+ */
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.tools;
+
+import java.io.File;
+import java.io.UnsupportedEncodingException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.tools.bean.ExportOrg;
+
+
+
+import java.io.File;
+import java.io.UnsupportedEncodingException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.UUID;
+
+import org.codehaus.jackson.JsonEncoding;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.Results.Level;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.tools.bean.ExportOrg;
+import org.apache.usergrid.utils.JsonUtils;
+
+import org.apache.commons.cli.CommandLine;
+
+import com.google.common.collect.BiMap;
+
+
+public class ExportAdmins extends ExportingToolBase {
+
+    static final Logger logger = LoggerFactory.getLogger( Export.class );
+
+    JsonFactory jsonFactory = new JsonFactory();
+
+
+    @Override
+    public void runTool( CommandLine line ) throws Exception {
+        startSpring();
+
+        setVerbose( line );
+
+        // ExportDataCreator dataCreator = new ExportDataCreator(emf,
+        // managementService);
+        // dataCreator.createTestData();
+
+        applyOrgId( line );
+        prepareBaseOutputFileName( line );
+        outputDir = createOutputParentDir();
+        logger.info( "Export directory: " + outputDir.getAbsolutePath() );
+
+        // Export organizations separately.
+//        exportOrganizations();
+//
+//        // Loop through the organizations
+//        Map<UUID, String> organizations = getOrgs();
+//        for ( Map.Entry<UUID, String> organization : organizations.entrySet() ) {
+//
+//            if ( organization.equals( properties.getProperty( "usergrid.test-account.organization"
) ) ) {
+//                // Skip test data from being exported.
+//                continue;
+//            }
+
+            exportApplicationsForOrg( null );
+     //   }
+    }
+
+
+    private Map<UUID, String> getOrgs() throws Exception {
+        // Loop through the organizations
+        Map<UUID, String> organizationNames = null;
+
+        if ( orgId == null ) {
+            organizationNames = managementService.getOrganizations();
+        }
+
+        else {
+            OrganizationInfo info = managementService.getOrganizationByUuid( orgId );
+
+            if ( info == null ) {
+                logger.error( "Organization info is null!" );
+                System.exit( 1 );
+            }
+
+            organizationNames = new HashMap<UUID, String>();
+            organizationNames.put( orgId, info.getName() );
+        }
+
+
+        return organizationNames;
+    }
+
+
+    private void exportApplicationsForOrg( Map.Entry<UUID, String> organization ) throws
Exception {
+        logger.info( "" + organization );
+
+        // Loop through the applications per organization
+        //BiMap<UUID, String> applications = managementService.getApplicationsForOrganization(
organization.getKey() );
+        //for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
+
+          //  logger.info( application.getValue() + " : " + application.getKey() );
+
+            // Get the JSon serializer.
+            JsonGenerator jg = getJsonGenerator( createOutputFile( "application", "admin_user")
);
+//
+//            // load the dictionary
+//            EntityManager rootEm = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID
);
+//
+//            Entity appEntity = rootEm.get( application.getKey() );
+//
+//            Map<String, Object> dictionaries = new HashMap<String, Object>();
+//
+//            for ( String dictionary : rootEm.getDictionaries( appEntity ) ) {
+//                Map<Object, Object> dict = rootEm.getDictionaryAsMap( appEntity,
dictionary );
+//
+//                // nothing to do
+//                if ( dict.isEmpty() ) {
+//                    continue;
+//                }
+//
+//                dictionaries.put( dictionary, dict );
+//            }
+////
+            EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID
);
+//
+//            // Get application
+//            Entity nsEntity = em.get( application.getKey() );
+//
+//            Set<String> collections = em.getApplicationCollections();
+//
+//            // load app counters
+//
+//            Map<String, Long> entityCounters = em.getApplicationCounters();
+//
+//            nsEntity.setMetadata( "organization", organization );
+//            nsEntity.setMetadata( "dictionaries", dictionaries );
+//            // counters for collections
+//            nsEntity.setMetadata( "counters", entityCounters );
+//            nsEntity.setMetadata( "collections", collections );
+
+              jg.writeStartArray();
+//            jg.writeObject( nsEntity );
+
+            // Create a GENERATOR for the application collections.
+            JsonGenerator collectionsJg = getJsonGenerator( createOutputFile( "collections",
"admin_users" ) );
+            collectionsJg.writeStartObject();
+
+            Map<String, Object> metadata = em.getApplicationCollectionMetadata();
+            echo( JsonUtils.mapToFormattedJsonString( metadata ) );
+
+            // Loop through the collections. This is the only way to loop
+            // through the entities in the application (former namespace).
+           // for ( String collectionName : metadata.keySet() ) {
+
+                Query query = new Query();
+                query.setLimit( MAX_ENTITY_FETCH );
+                query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+
+                Results entities = em.searchCollection( em.getApplicationRef(), "users",
query );
+
+                while ( entities.size() > 0 ) {
+
+                    for ( Entity entity : entities ) {
+                        // Export the entity first and later the collections for
+                        // this entity.
+                        jg.writeObject( entity );
+                        echo( entity );
+
+                        saveCollectionMembers( collectionsJg, em, null, entity );
+                    }
+
+                    //we're done
+                    if ( entities.getCursor() == null ) {
+                        break;
+                    }
+
+
+                    query.setCursor( entities.getCursor() );
+
+                    entities = em.searchCollection( em.getApplicationRef(), "users", query
);
+                }
+            //}
+
+            // Close writer for the collections for this application.
+            collectionsJg.writeEndObject();
+            collectionsJg.close();
+
+            // Close writer and file for this application.
+            jg.writeEndArray();
+            jg.close();
+       // }
+    }
+
+
+    /**
+     * Serialize and save the collection members of this <code>entity</code>
+     *
+     * @param em Entity Manager
+     * @param application Application name
+     * @param entity entity
+     */
+    private void saveCollectionMembers( JsonGenerator jg, EntityManager em, String application,
Entity entity )
+            throws Exception {
+
+        Set<String> collections = em.getCollections( entity );
+
+        // Only create entry for Entities that have collections
+        if ( ( collections == null ) || collections.isEmpty() ) {
+            return;
+        }
+
+        jg.writeFieldName( entity.getUuid().toString() );
+        jg.writeStartObject();
+
+        for ( String collectionName : collections ) {
+
+            jg.writeFieldName( collectionName );
+            // Start collection array.
+            jg.writeStartArray();
+
+            Results collectionMembers = em.getCollection( entity, collectionName, null, 100000,
Level.IDS, false );
+
+            List<UUID> entityIds = collectionMembers.getIds();
+
+            if ( ( entityIds != null ) && !entityIds.isEmpty() ) {
+                for ( UUID childEntityUUID : entityIds ) {
+                    jg.writeObject( childEntityUUID.toString() );
+                }
+            }
+
+            // End collection array.
+            jg.writeEndArray();
+        }
+
+        // Write connections
+        saveConnections( entity, em, jg );
+
+        // Write dictionaries
+        saveDictionaries( entity, em, jg );
+
+        // End the object if it was Started
+        jg.writeEndObject();
+    }
+
+
+    /** Persists the connection for this entity. */
+    private void saveDictionaries( Entity entity, EntityManager em, JsonGenerator jg ) throws
Exception {
+
+        jg.writeFieldName( "dictionaries" );
+        jg.writeStartObject();
+
+        Set<String> dictionaries = em.getDictionaries( entity );
+        for ( String dictionary : dictionaries ) {
+
+            Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary );
+
+            // nothing to do
+            if ( dict.isEmpty() ) {
+                continue;
+            }
+
+            jg.writeFieldName( dictionary );
+
+            jg.writeStartObject();
+
+            for ( Map.Entry<Object, Object> entry : dict.entrySet() ) {
+                jg.writeFieldName( entry.getKey().toString() );
+                jg.writeObject( entry.getValue() );
+            }
+
+            jg.writeEndObject();
+        }
+        jg.writeEndObject();
+    }
+
+
+    /** Persists the connection for this entity. */
+    private void saveConnections( Entity entity, EntityManager em, JsonGenerator jg ) throws
Exception {
+
+        jg.writeFieldName( "connections" );
+        jg.writeStartObject();
+
+        Set<String> connectionTypes = em.getConnectionTypes( entity );
+        for ( String connectionType : connectionTypes ) {
+
+            jg.writeFieldName( connectionType );
+            jg.writeStartArray();
+
+            Results results = em.getConnectedEntities( entity.getUuid(), connectionType,
null, Level.IDS );
+            List<ConnectionRef> connections = results.getConnections();
+
+            for ( ConnectionRef connectionRef : connections ) {
+                jg.writeObject( connectionRef.getConnectedEntity().getUuid() );
+            }
+
+            jg.writeEndArray();
+        }
+        jg.writeEndObject();
+    }
+
+  /*-
+   * Set<String> collections = em.getCollections(entity);
+   * for (String collection : collections) {
+   *   Results collectionMembers = em.getCollection(
+   *    entity, collection, null,
+   *    MAX_ENTITY_FETCH, Level.IDS, false);
+   *    write entity_id : { "collectionName" : [ids]
+   *  }
+   * }
+   *
+   *
+   *   {
+   *     entity_id :
+   *       { collection_name :
+   *         [
+   *           collected_entity_id,
+   *           collected_entity_id
+   *         ]
+   *       },
+   *     f47ac10b-58cc-4372-a567-0e02b2c3d479 :
+   *       { "activtites" :
+   *         [
+   *           f47ac10b-58cc-4372-a567-0e02b2c3d47A,
+   *           f47ac10b-58cc-4372-a567-0e02b2c3d47B
+   *         ]
+   *       }
+   *   }
+   *
+   * http://jackson.codehaus.org/1.8.0/javadoc/org/codehaus/jackson/JsonGenerator.html
+   *
+   *
+   *-
+   * List<ConnectedEntityRef> connections = em.getConnections(entityId, query);
+   */
+
+
+    private void exportOrganizations() throws Exception, UnsupportedEncodingException {
+
+
+        for ( Map.Entry<UUID, String> organizationName : getOrgs().entrySet() ) {
+
+            // Let's skip the test entities.
+            if ( organizationName.equals( properties.getProperty( "usergrid.test-account.organization"
) ) ) {
+                continue;
+            }
+
+            OrganizationInfo acc = managementService.getOrganizationByUuid( organizationName.getKey()
);
+            logger.info( "Exporting Organization: " + acc.getName() );
+
+            ExportOrg exportOrg = new ExportOrg( acc );
+
+            List<UserInfo> users = managementService.getAdminUsersForOrganization(
organizationName.getKey() );
+
+            for ( UserInfo user : users ) {
+                exportOrg.addAdmin( user.getUsername() );
+            }
+
+            // One file per Organization.
+            saveOrganizationInFile( exportOrg );
+        }
+    }
+
+
+    /**
+     * Serialize an Organization into a json file.
+     *
+     * @param acc OrganizationInfo
+     */
+    private void saveOrganizationInFile( ExportOrg acc ) {
+        try {
+
+            File outFile = createOutputFile( "organization", acc.getName() );
+            JsonGenerator jg = getJsonGenerator( outFile );
+            jg.writeObject( acc );
+            jg.close();
+        }
+        catch ( Exception e ) {
+            throw new RuntimeException( e );
+        }
+    }
+
+
+    public void streamOutput( File file, List<Entity> entities ) throws Exception {
+        JsonFactory jsonFactory = new JsonFactory();
+        // or, for data binding,
+        // org.codehaus.jackson.mapper.MappingJsonFactory
+        JsonGenerator jg = jsonFactory.createJsonGenerator( file, JsonEncoding.UTF8 );
+        // or Stream, Reader
+
+        jg.writeStartArray();
+        for ( Entity entity : entities ) {
+            jg.writeObject( entity );
+        }
+        jg.writeEndArray();
+
+        jg.close();
+    }
+
+    // to generate the activities and user relationship, follow this:
+
+    // write field name (id)
+    // write start object
+    // write field name (collection name)
+    // write start array
+    // write object/string
+    // write another object
+    // write end array
+    // write end object
+    // ...... more objects
+    //
+}
+


Mime
View raw message