usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From g...@apache.org
Subject incubator-usergrid git commit: Merged in changes to that Dave made to the exportAdmins script due to this having most of the same underlying code.
Date Wed, 08 Jul 2015 17:22:05 GMT
Repository: incubator-usergrid
Updated Branches:
  refs/heads/MigrationTool d5573059c -> a03d36b46


Merged in changes to that Dave made to the exportAdmins script due to this having most of
the same underlying code.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a03d36b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a03d36b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a03d36b4

Branch: refs/heads/MigrationTool
Commit: a03d36b46f0c49e5150b7deede47c36be080a855
Parents: d557305
Author: GERey <greyes@apigee.com>
Authored: Wed Jul 8 10:22:02 2015 -0700
Committer: GERey <greyes@apigee.com>
Committed: Wed Jul 8 10:22:02 2015 -0700

----------------------------------------------------------------------
 .../org/apache/usergrid/tools/Migration.java    | 124 ++++++++++++-------
 1 file changed, 77 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a03d36b4/stack/tools/src/main/java/org/apache/usergrid/tools/Migration.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Migration.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Migration.java
index 42f6855..0f747bb 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/Migration.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Migration.java
@@ -64,7 +64,7 @@ public class Migration extends ExportingToolBase {
      */
     class ApplicationWriteTask {
         Entity entity;
-        Map<String, List<UUID>>          collectionsByName;
+        //Map<String, List<UUID>>          collectionsByName;
         Map<String, List<ConnectionRef>> connectionsByType;
         Map<String, Map<Object, Object>> dictionariesByName;
     }
@@ -181,13 +181,18 @@ public class Migration extends ExportingToolBase {
             }
         }
 
-        applicationWriter.setDone( true );
-        for (ApplicationReader aur : readers) {
-            aur.setDone( true );
-        }
+//        applicationWriter.setDone( true );
+//        for (ApplicationReader aur : readers) {
+//            aur.setDone( true );
+//        }
 
         logger.debug( "Waiting for write thread to complete" );
-        writeThread.join();
+        boolean done = false;
+        while ( !done ) {
+            writeThread.join( 10000, 0 );
+            done = !writeThread.isAlive();
+            //logger.info( "Wrote {} users", count.get() );
+        }
     }
 
     public EntityManager applicationEntityManagerCreator() throws Exception{
@@ -223,8 +228,6 @@ public class Migration extends ExportingToolBase {
 
     public class ApplicationReader implements Runnable {
 
-        private boolean done = true;
-
         private final BlockingQueue<UUID> readQueue;
         private final BlockingQueue<ApplicationWriteTask> writeQueue;
 
@@ -255,7 +258,7 @@ public class Migration extends ExportingToolBase {
                     uuid = readQueue.poll( 30, TimeUnit.SECONDS );
                     logger.debug("Got item from entityId queue: " + uuid );
 
-                    if ( uuid == null && done ) {
+                    if ( uuid == null ) {
                         break;
                     }
 
@@ -277,49 +280,85 @@ public class Migration extends ExportingToolBase {
         }
 
 
-        private void addCollectionsToTask(ApplicationWriteTask task, Entity entity) throws
Exception {
+//        private void addCollectionsToTask(ApplicationWriteTask task, Entity entity) throws
Exception {
+//
+//            EntityManager em = applicationEntityManagerCreator();
+//            Set<String> collections = em.getCollections( entity );
+//            if ((collections == null) || collections.isEmpty()) {
+//                return;
+//            }
+//
+//            task.collectionsByName = new HashMap<String, List<UUID>>();
+//
+//            for (String collectionName : collections) {
+//
+//                List<UUID> uuids = task.collectionsByName.get( collectionName );
+//                if ( uuids == null ) {
+//                    uuids = new ArrayList<UUID>();
+//                    task.collectionsByName.put( collectionName, uuids );
+//                }
+//
+//                //probably better to read sets of a thousand rather then a single bulk
100k pull.
+//                Results collectionMembers = em.getCollection( entity, collectionName, null,
100000, Level.IDS, false );
+//
+//                List<UUID> entityIds = collectionMembers.getIds();
+//
+//                if ((entityIds != null) && !entityIds.isEmpty()) {
+//                    for (UUID childEntityUUID : entityIds) {
+//                        uuids.add( childEntityUUID );
+//                    }
+//                }
+//            }
+//        }
 
-            EntityManager em = applicationEntityManagerCreator();
-            Set<String> collections = em.getCollections( entity );
-            if ((collections == null) || collections.isEmpty()) {
-                return;
-            }
 
-            task.collectionsByName = new HashMap<String, List<UUID>>();
+        private void addDictionariesToTask(ApplicationWriteTask task, Entity entity) throws
Exception {
+            EntityManager em = applicationEntityManagerCreator();
 
-            for (String collectionName : collections) {
+            Set<String> dictionaries = em.getDictionaries( entity );
 
-                List<UUID> uuids = task.collectionsByName.get( collectionName );
-                if ( uuids == null ) {
-                    uuids = new ArrayList<UUID>();
-                    task.collectionsByName.put( collectionName, uuids );
-                }
+//            task.dictionariesByName = new HashMap<String, Map<Object, Object>>();
+//
+//            for (String dictionary : dictionaries) {
+//                Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary
);
+//                task.dictionariesByName.put( dictionary, dict );
+//            }
 
-                //probably better to read sets of a thousand rather then a single bulk 100k
pull.
-                Results collectionMembers = em.getCollection( entity, collectionName, null,
100000, Level.IDS, false );
 
-                List<UUID> entityIds = collectionMembers.getIds();
 
-                if ((entityIds != null) && !entityIds.isEmpty()) {
-                    for (UUID childEntityUUID : entityIds) {
-                        uuids.add( childEntityUUID );
-                    }
-                }
+            if ( dictionaries.isEmpty() ) {
+                logger.error("User {}:{} has no dictionaries", task.entity.getName(), task.entity.getUuid()
);
             }
-        }
-
-
-        private void addDictionariesToTask(ApplicationWriteTask task, Entity entity) throws
Exception {
-            EntityManager em = applicationEntityManagerCreator();
-
-            Set<String> dictionaries = em.getDictionaries( entity );
 
             task.dictionariesByName = new HashMap<String, Map<Object, Object>>();
 
             for (String dictionary : dictionaries) {
                 Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary
);
+                if ( dict.isEmpty() ) {
+                    continue;
+                }
                 task.dictionariesByName.put( dictionary, dict );
             }
+
+            if ( task.dictionariesByName.isEmpty() ) {
+                logger.error( "User {}:{} has no dictionaries",
+                        new Object[]{task.entity.getName(), task.entity.getUuid() } );
+
+            } else if ( task.dictionariesByName.get("credentials") == null ) {
+                logger.error( "User {}:{} has no credentials dictionary",
+                        new Object[]{task.entity.getName(), task.entity.getUuid() } );
+
+            } else {
+                if ( task.dictionariesByName.get("credentials").get("password") == null )
{
+                    logger.error( "User {}:{} has no password in credential dictionary",
+                            new Object[]{task.entity.getName(), task.entity.getUuid() } );
+                }
+                if ( task.dictionariesByName.get("credentials").get("secret") == null ) {
+                    logger.error( "User {}:{} has no secret in credential dictionary",
+                            new Object[]{task.entity.getName(), task.entity.getUuid() } );
+                }
+            }
+
         }
 
 
@@ -345,15 +384,10 @@ public class Migration extends ExportingToolBase {
             }
         }
 
-        public void setDone(boolean done) {
-            this.done = done;
-        }
     }
 
     class ApplicationWriter implements Runnable {
 
-        private boolean done = false;
-
         private final BlockingQueue<ApplicationWriteTask> taskQueue;
 
         public ApplicationWriter( BlockingQueue<ApplicationWriteTask> taskQueue ) {
@@ -389,7 +423,7 @@ public class Migration extends ExportingToolBase {
 
                 try {
                     ApplicationWriteTask task = taskQueue.poll( 30, TimeUnit.SECONDS );
-                    if ( task == null && done ) {
+                    if ( task == null ) {
                         break;
                     }
 
@@ -477,10 +511,6 @@ public class Migration extends ExportingToolBase {
             }
             jg.writeEndObject();
         }
-
-        public void setDone(boolean done) {
-            this.done = done;
-        }
     }
 }
 


Mime
View raw message