usergrid-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mru...@apache.org
Subject [09/35] usergrid git commit: add deletion of orphans to collection iterator
Date Fri, 02 Sep 2016 00:51:28 GMT
add deletion of orphans to collection iterator


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/cda2879c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/cda2879c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/cda2879c

Branch: refs/heads/master
Commit: cda2879c03d64c5d30565fb037d5b6588a8b793c
Parents: fd7a75e
Author: Mike Dunker <mdunker@apigee.com>
Authored: Fri Aug 26 16:40:38 2016 -0700
Committer: Mike Dunker <mdunker@apigee.com>
Committed: Fri Aug 26 16:40:38 2016 -0700

----------------------------------------------------------------------
 .../usergrid/tools/CollectionIterator.java      | 107 ++++++++++++++-----
 1 file changed, 81 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/cda2879c/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
index 855c363..a682e44 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
@@ -20,31 +20,19 @@ package org.apache.usergrid.tools;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
 
-import com.google.common.base.*;
 import com.google.common.base.Optional;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import com.netflix.astyanax.model.Column;
-import com.netflix.astyanax.model.ConsistencyLevel;
-import com.netflix.astyanax.util.RangeBuilder;
 import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
-import org.apache.usergrid.corepersistence.results.EntityQueryExecutor;
 import org.apache.usergrid.corepersistence.results.IdQueryExecutor;
 import org.apache.usergrid.corepersistence.service.CollectionSearch;
 import org.apache.usergrid.corepersistence.service.CollectionService;
 import org.apache.usergrid.persistence.*;
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
-import org.apache.usergrid.persistence.collection.serialization.impl.*;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
 import org.apache.usergrid.persistence.index.utils.UUIDUtils;
 import org.apache.usergrid.persistence.model.entity.*;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.field.StringField;
 import org.apache.usergrid.persistence.schema.CollectionInfo;
 import org.apache.usergrid.utils.InflectionUtils;
+import static org.apache.commons.lang.StringUtils.isBlank;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -54,11 +42,6 @@ import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 
 
-import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily;
-import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
-import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
-import rx.*;
-
 import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
 
 
@@ -72,6 +55,14 @@ public class CollectionIterator extends ToolBase {
 
     private static final String REMOVE_CONNECTIONS_ARG = "removeConnections";
 
+    private static final String LATEST_TIMESTAMP_ARG = "latestTimestamp";
+
+    private static final String EARLIEST_TIMESTAMP_ARG = "earliestTimestamp";
+
+    private static final String SECONDS_IN_PAST_ARG = "secondsInPast";
+
+    private static final Long DEFAULT_SECONDS_IN_PAST = 60L * 60L; // hour
+
     private EntityManager em;
 
     @Override
@@ -95,11 +86,29 @@ public class CollectionIterator extends ToolBase {
         options.addOption( collectionOption );
 
         Option removeConnectionsOption =
-            OptionBuilder.withArgName(REMOVE_CONNECTIONS_ARG).hasArg().isRequired( false
).withDescription( "remove orphaned connections" )
-                .create(REMOVE_CONNECTIONS_ARG);
+                OptionBuilder.withArgName(REMOVE_CONNECTIONS_ARG).hasArg().isRequired( false
).withDescription( "remove orphaned connections" )
+                        .create(REMOVE_CONNECTIONS_ARG);
 
         options.addOption( removeConnectionsOption );
 
+        Option earliestTimestampOption =
+                OptionBuilder.withArgName(EARLIEST_TIMESTAMP_ARG).hasArg().isRequired( false
).withDescription( "earliest timestamp to delete" )
+                        .create(EARLIEST_TIMESTAMP_ARG);
+
+        options.addOption( earliestTimestampOption );
+
+        Option latestTimestampOption =
+                OptionBuilder.withArgName(LATEST_TIMESTAMP_ARG).hasArg().isRequired( false
).withDescription( "latest timestamp to delete" )
+                        .create(LATEST_TIMESTAMP_ARG);
+
+        options.addOption( latestTimestampOption );
+
+        Option secondsInPastOption =
+            OptionBuilder.withArgName(SECONDS_IN_PAST_ARG).hasArg().isRequired( false ).withDescription(
"how many seconds old orphan must be to be deleted" )
+                .create(SECONDS_IN_PAST_ARG);
+
+        options.addOption( secondsInPastOption );
+
 
         return options;
     }
@@ -116,18 +125,62 @@ public class CollectionIterator extends ToolBase {
 
         startSpring();
 
-        if (line.getOptionValue(APPLICATION_ARG).isEmpty()) {
+        String applicationOption = line.getOptionValue(APPLICATION_ARG);
+        String entityTypeOption = line.getOptionValue(ENTITY_TYPE_ARG);
+        String removeConnectionsOption = line.getOptionValue(REMOVE_CONNECTIONS_ARG);
+        String earliestTimestampOption = line.getOptionValue(EARLIEST_TIMESTAMP_ARG);
+        String latestTimestampOption = line.getOptionValue(LATEST_TIMESTAMP_ARG);
+        String secondsInPastOption = line.getOptionValue(SECONDS_IN_PAST_ARG);
+
+        if (isBlank(applicationOption)) {
             throw new RuntimeException("Application ID not provided.");
         }
         final UUID app = UUID.fromString(line.getOptionValue(APPLICATION_ARG));
 
-        String removeOrphansOption = line.getOptionValue(REMOVE_CONNECTIONS_ARG);
-        final boolean removeOrphans = !removeOrphansOption.isEmpty() && removeOrphansOption.toLowerCase().equals("yes");
+        if (isBlank(entityTypeOption)) {
+            throw new RuntimeException("Entity type (singular collection name) not provided.");
+        }
+        String entityType = entityTypeOption;
+
+        final boolean removeOrphans = !isBlank(removeConnectionsOption) && removeConnectionsOption.toLowerCase().equals("yes");
+
+        if (!isBlank(secondsInPastOption) && !isBlank(latestTimestampOption)) {
+            throw new RuntimeException("Can't specify both latest timestamp and seconds in
past options.");
+        }
 
-        String entityType = line.getOptionValue(ENTITY_TYPE_ARG);
+        long earliest = 0L;
+        if (!isBlank(earliestTimestampOption)) {
+            try {
+                earliest = Long.parseLong(earliestTimestampOption);
+            } catch (Exception e) {
+                throw new RuntimeException("Cannot convert earliest timestamp to long: "
+ earliestTimestampOption);
+            }
+        }
+        final long earliestTimestamp = earliest;
+
+        long currentTimestamp = System.currentTimeMillis();
+
+        // default to DEFAULT_SECONDS_IN_PAST
+        long latest = currentTimestamp - (DEFAULT_SECONDS_IN_PAST * 1000L);
+        if (!isBlank(latestTimestampOption)) {
+            try {
+                latest = Long.parseLong(latestTimestampOption);
+            } catch (Exception e) {
+                throw new RuntimeException("Cannot convert latest timestamp to long: " +
latestTimestampOption);
+            }
+        } else if (!isBlank(secondsInPastOption)) {
+            try {
+                long secondsInPast = Long.parseLong(secondsInPastOption);
+                latest = currentTimestamp - (secondsInPast * 1000L);
+            } catch (Exception e) {
+                throw new RuntimeException("Cannot convert seconds in past to long: " + secondsInPastOption);
+            }
+        }
+        final long latestTimestamp = latest;
 
         logger.info("Starting Tool: CollectionIterator");
         logger.info("Orphans {} be deleted", removeOrphans ? "WILL" : "will not");
+        logger.info("Timestamp range {} to {}", Long.toString(earliestTimestamp), Long.toString(latestTimestamp));
         logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl",
"CL_LOCAL_QUORUM"));
 
         em = emf.getEntityManager( app );
@@ -184,15 +237,17 @@ public class CollectionIterator extends ToolBase {
 
                         logger.info("{} - {} - entity data found", uuid, dateString);
                     }else{
-                        if (removeOrphans) {
+                        if (removeOrphans && timestamp >= earliestTimestamp &&
timestamp <= latestTimestamp) {
                             logger.info("{} - {} - entity data NOT found, REMOVING", uuid,
dateString);
                             try {
                                 em.removeFromCollection(headEntity, collectionName, entityRef
);
                             } catch (Exception e) {
                                 logger.error("{} - exception while trying to remove orphaned
connection, {}", uuid, e.getMessage());
                             }
+                        } else if (removeOrphans) {
+                            logger.info("{} - {} ({}) - entity data NOT found, not removing
because timestamp not in range", uuid, dateString, timestamp);
                         } else {
-                            logger.info("{} - {} - entity data NOT found", uuid, dateString);
+                            logger.info("{} - {} ({}) - entity data NOT found", uuid, dateString,
timestamp);
                         }
                     }
                 } catch (Exception e) {


Mime
View raw message