usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mru...@apache.org
Subject [2/7] usergrid git commit: Add double protection for dryrun mode, and some more logging.
Date Thu, 17 Dec 2015 22:14:21 GMT
Add double protection for dryrun mode, and some more logging.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/de254f0a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/de254f0a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/de254f0a

Branch: refs/heads/1.x
Commit: de254f0acce9f69842f04be3fc0904dae37f54a4
Parents: 674532f
Author: Dave Johnson <snoopdave@apache.org>
Authored: Fri Dec 11 14:54:26 2015 -0500
Committer: Dave Johnson <snoopdave@apache.org>
Committed: Fri Dec 11 14:54:26 2015 -0500

----------------------------------------------------------------------
 .../usergrid/tools/DuplicateOrgRepair.java      | 102 +++++++++++++++----
 .../usergrid/tools/DuplicateOrgRepairTest.java  |   6 ++
 2 files changed, 87 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/de254f0a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
index bc4acb5..4b5e391 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
@@ -18,7 +18,6 @@
 package org.apache.usergrid.tools;
 
 import com.google.common.collect.BiMap;
-import com.google.common.collect.Sets;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
@@ -51,7 +50,7 @@ import static org.apache.usergrid.tools.DuplicateOrgInterface.OrgUser;
  */
 public class DuplicateOrgRepair extends ToolBase {
 
-    DuplicateOrgInterface   manager = new Manager();
+    DuplicateOrgInterface   manager;
     
     Map<String, Set<Org>>   orgsByName = new HashMap<String, Set<Org>>();
     
@@ -68,6 +67,8 @@ public class DuplicateOrgRepair extends ToolBase {
     static final String     DRYRUN_ARG_NAME = "dryrun";
     
     static boolean          dryRun = false;
+    
+    static boolean          testing = false;
 
     
     @Override
@@ -94,13 +95,10 @@ public class DuplicateOrgRepair extends ToolBase {
     @Override
     public void runTool(CommandLine line) throws Exception {
 
-        logger.info( "DuplicateOrgRepair tool starting up..." );
 
         startSpring();
         setVerbose( line );
 
-        dryRun = Boolean.parseBoolean( line.getOptionValue( DRYRUN_ARG_NAME ));
-
         if (StringUtils.isNotEmpty( line.getOptionValue( THREADS_ARG_NAME ) )) {
             try {
                 threadCount = Integer.parseInt( line.getOptionValue( THREADS_ARG_NAME ) );
@@ -110,15 +108,27 @@ public class DuplicateOrgRepair extends ToolBase {
             }
         }
 
-        buildOrgMaps();
+        dryRun = Boolean.parseBoolean( line.getOptionValue( DRYRUN_ARG_NAME ));
+        
+        if ( dryRun && !testing ) {
+            manager = new DryRunManager();
+        } else {
+            manager = new RepairManager();
+        }
 
+        logger.info( "DuplicateOrgRepair tool starting up... manager: " + manager.getClass().getSimpleName()
);
+        
+        buildOrgMaps();
+        
         augmentUserOrgsMap();
         
+        manager.logDuplicates( duplicatesByName );
+        
         mergeDuplicateOrgs();
-
+        
         removeDuplicateOrgs();
 
-        logger.info( "DuplicateOrgRepair work is done." );
+        logger.info( "DuplicateOrgRepair work is done!");
     }
 
     
@@ -158,10 +168,14 @@ public class DuplicateOrgRepair extends ToolBase {
                     }
                 } catch (Exception e) {
                     logger.error("Error getting users for org {}:{}", org.getName(), org.getId());
+                    logger.error("Stack trace is: ", e);
                 }
 
             }
+            
         } ).toBlocking().lastOrDefault( null );
+        
+        logger.info( "DuplicateOrgRepair tool built org maps");
     }
 
     
@@ -179,6 +193,9 @@ public class DuplicateOrgRepair extends ToolBase {
                 try {
                     Set<Org> connectedToOrgs = manager.getUsersOrgs(user);
                     Set<Org> usersOrgs = orgsByUser.get(user);
+                    if ( usersOrgs == null ) {
+                        usersOrgs = new HashSet<Org>();
+                    }
                     for ( Org org : connectedToOrgs ) {
                         if (!usersOrgs.contains(org)) {
                             usersOrgs.add(org);
@@ -187,9 +204,12 @@ public class DuplicateOrgRepair extends ToolBase {
 
                 } catch (Exception e) {
                     logger.error("Error getting orgs for user {}:{}", user.getName(), user.getId());
+                    logger.error("Stack trace is: ", e);
                 }
             }
         } ).subscribeOn( scheduler ).toBlocking().lastOrDefault( null );
+
+        logger.info( "DuplicateOrgRepair augmented user orgs map"); 
     }
 
 
@@ -216,10 +236,10 @@ public class DuplicateOrgRepair extends ToolBase {
                             args = new Object[] { 
                                     user.getName(), user.getId(), org.getName(), org.getId()};
                             logger.info( "Would remove user {}:{}  org {}:{}", args);
-                            continue;
-                        } 
-                        manager.addUserToOrg( user, bestOrg );
-                        manager.removeUserFromOrg( user, org );
+                        } else {
+                            manager.addUserToOrg( user, bestOrg );
+                            manager.removeUserFromOrg( user, org );
+                        }
                     }
                     
                     Set<UUID> orgApps = new HashSet<UUID>( manager.getOrgApps(
org ));
@@ -232,14 +252,16 @@ public class DuplicateOrgRepair extends ToolBase {
                             args = new Object[] { 
                                     appId, org.getName(), org.getId()};
                             logger.info( "Would remove app {} org {}:{}", args);
-                            continue;
-                        } 
-                        manager.addAppToOrg( appId, bestOrg );
-                        manager.removeAppFromOrg( appId, org );
+                        } else {
+                            manager.addAppToOrg( appId, bestOrg );
+                            manager.removeAppFromOrg( appId, org );
+                        }
                     }
                 }
             }
         }
+
+        logger.info( "DuplicateOrgRepair merged duplicate orgs"); 
     }
 
 
@@ -261,6 +283,7 @@ public class DuplicateOrgRepair extends ToolBase {
                 }
             }
         }
+        logger.info( "DuplicateOrgRepair renamed/removed duplicate orgs"); 
     }
 
 
@@ -276,9 +299,11 @@ public class DuplicateOrgRepair extends ToolBase {
         }
         return oldest;
     }
-    
 
-    class Manager implements DuplicateOrgInterface {
+
+
+    
+    class RepairManager implements DuplicateOrgInterface {
 
         private boolean dryRun = true;
 
@@ -291,6 +316,8 @@ public class DuplicateOrgRepair extends ToolBase {
                 public void call(Subscriber<? super Org> subscriber) {
                     subscriber.onStart();
                     try {
+                        int count = 0;
+                        
                         Query query = new Query();
                         query.setLimit( MAX_ENTITY_FETCH );
                         query.setResultsLevel( Results.Level.ALL_PROPERTIES );
@@ -308,6 +335,10 @@ public class DuplicateOrgRepair extends ToolBase {
                                 
                                 subscriber.onNext( org );
 
+                                if ( count++ % 1000 == 0 ) {
+                                    logger.info("Emitted {} orgs", count );
+                                }
+
                                 // logger.info( "org: {}, \"{}\", {}", new Object[]{
                                 //     orgEntity.getProperty( "path" ),
                                 //     orgEntity.getUuid(),
@@ -337,6 +368,8 @@ public class DuplicateOrgRepair extends ToolBase {
                 public void call(Subscriber<? super OrgUser> subscriber) {
                     subscriber.onStart();
                     try {
+                        int count = 0;
+                        
                         Query query = new Query();
                         query.setLimit( MAX_ENTITY_FETCH );
                         query.setResultsLevel( Results.Level.ALL_PROPERTIES );
@@ -352,7 +385,11 @@ public class DuplicateOrgRepair extends ToolBase {
                                 orgUser.sourceValue = entity;
 
                                 subscriber.onNext( orgUser );
-                                
+
+                                if ( count++ % 1000 == 0 ) {
+                                    logger.info("Emitted {} users", count );
+                                }
+
                                 // logger.info( "org: {}, \"{}\", {}", new Object[]{
                                 //     entity.getProperty( "path" ),
                                 //     entity.getUuid(),
@@ -401,9 +438,7 @@ public class DuplicateOrgRepair extends ToolBase {
         
         @Override
         public void removeOrg(Org keeper, Org duplicate) throws Exception {
-
             // we don't have a remove org API so rename org so that it is no longer a duplicate
-            
             OrganizationInfo orgInfo = managementService.getOrganizationByUuid( duplicate.getId()
);
             orgInfo.setName( "dup_" + keeper.getId() + "_" + RandomStringUtils.randomAlphanumeric(10)
);
             managementService.updateOrganization( orgInfo );
@@ -491,4 +526,29 @@ public class DuplicateOrgRepair extends ToolBase {
         }
         
     }
+
+    
+    class DryRunManager extends RepairManager {
+        
+        @Override
+        public void removeUserFromOrg(OrgUser user, Org org) throws Exception {
+        }
+
+        @Override
+        public void addUserToOrg(OrgUser user, Org org) throws Exception {
+        }
+
+        @Override
+        public void addAppToOrg(UUID appId, Org org) throws Exception {
+        }
+
+        @Override
+        public void removeAppFromOrg(UUID appId, Org org) throws Exception {
+        }
+
+        @Override
+        public void removeOrg(Org keeper, Org duplicate) throws Exception {
+        }
+    }
+    
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/de254f0a/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
b/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
index ac95748..4c92761 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
@@ -57,6 +57,8 @@ public class DuplicateOrgRepairTest {
         int numOrgs = 10; // create 10 orgs and a dup for each
 
         final DuplicateOrgRepair dor = new DuplicateOrgRepair();
+        dor.testing = true;
+        
         dor.manager = new Manager( numOrgs );
 
         assertEquals( "must start with dups", 
@@ -92,6 +94,7 @@ public class DuplicateOrgRepairTest {
         int numOrgs = 10; // create 10 orgs and a dup for each
 
         DuplicateOrgRepair dor = new DuplicateOrgRepair();
+        dor.testing = true;
         dor.manager = new Manager( numOrgs );
 
         assertEquals( "must start with dups", 
@@ -131,6 +134,7 @@ public class DuplicateOrgRepairTest {
                 orgOwnerInfo1.getOwner(), orgOwnerInfo2.getOrganization(), false );
         
         DuplicateOrgRepair dor = new DuplicateOrgRepair();
+        dor.testing = true;
         
         dor.startTool( new String[] {}, false );  // false means do not call System.exit()
         
@@ -156,6 +160,7 @@ public class DuplicateOrgRepairTest {
                 "user_" + random2 + "@example.com", "password" );
 
         DuplicateOrgRepair dor = new DuplicateOrgRepair();
+        dor.testing = true;
 
         // start the tool so thaht Spring, Cassandra, etc/ gets initialized
         dor.startTool( new String[] { "-dryrun", "true" }, false ); // false means do not
call System.exit()
@@ -216,6 +221,7 @@ public class DuplicateOrgRepairTest {
             orgOwnerInfo2.getOrganization().getUuid(), "app_" + RandomStringUtils.randomAlphanumeric(
10 ));
 
         DuplicateOrgRepair dor = new DuplicateOrgRepair();
+        dor.testing = true;
 
         // start the tool so that Spring, Cassandra, etc/ gets initialized
         dor.startTool( new String[] { "-dryrun", "true" }, false ); // false means do not
call System.exit()


Mime
View raw message