allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From brond...@apache.org
Subject [allura] 01/03: [#8374] enhance task retry and related commands with filter options
Date Fri, 21 Aug 2020 20:08:29 GMT
This is an automated email from the ASF dual-hosted git repository.

brondsem pushed a commit to branch db/8374
in repository https://gitbox.apache.org/repos/asf/allura.git

commit a6d4a85342dec25e5f650385f27543e2aff0636f
Author: Dave Brondsema <dave@brondsema.net>
AuthorDate: Fri Aug 21 16:07:02 2020 -0400

    [#8374] enhance task retry and related commands with filter options
---
 Allura/allura/command/taskd.py       | 48 +++++++++++++++++++++++++++++++-----
 Allura/allura/model/monq_model.py    | 14 -----------
 Allura/allura/tests/test_commands.py |  8 ++++++
 3 files changed, 50 insertions(+), 20 deletions(-)

diff --git a/Allura/allura/command/taskd.py b/Allura/allura/command/taskd.py
index 1f5ec78..3a79cd5 100644
--- a/Allura/allura/command/taskd.py
+++ b/Allura/allura/command/taskd.py
@@ -18,6 +18,8 @@
 from __future__ import unicode_literals
 from __future__ import print_function
 from __future__ import absolute_import
+
+import re
 import logging
 import os
 import time
@@ -168,12 +170,25 @@ class TaskCommand(base.Command):
     summary = 'Task command'
     parser = base.Command.standard_parser(verbose=True)
     parser.add_option('-s', '--state', dest='state', default='ready',
-                      help='state of processes to examine')
+                      help='state of processes for "list" subcommand.  * means all')
     parser.add_option('-t', '--timeout', dest='timeout', type=int, default=60,
                       help='timeout (in seconds) for busy tasks')
+    parser.add_option('--filter-name-prefix', dest='filter_name_prefix', default=None,
+                      help='limit to task names starting with this.  Example allura.tasks.index_tasks.')
+    parser.add_option('--filter-result-regex', dest='filter_result_regex', default=None,
+                      help='limit to tasks with result matching this regex.  Example "pysolr"')
     min_args = 2
     max_args = None
-    usage = '<ini file> [list|retry|purge|timeout|commit]'
+    usage = '''<ini file> [list|retry|purge|timeout|commit]
+
+    list: list tasks with given --state value
+    retry: re-run tasks with error state
+    purge: remove all "complete" tasks with result_type "forget" (which is the default)
+    timeout: retry all busy tasks older than --timeout seconds (does not stop existing task)
+    commit: run a solr 'commit' as a background task
+
+    All subcommands except 'commit' can use the --filter-... options.
+    '''
 
     def command(self):
         self.basic_setup()
@@ -186,6 +201,15 @@ class TaskCommand(base.Command):
             commit=self._commit)
         tab[cmd]()
 
+    def _add_filters(self, q):
+        if self.options.filter_name_prefix:
+            q['task_name'] = {'$regex': r'^{}.*'.format(re.escape(self.options.filter_name_prefix))}
+        if self.options.filter_result_regex:
+            q['result'] = {'$regex': self.options.filter_result_regex}
+        if self.verbose:
+            print(q)
+        return q
+
     def _list(self):
         '''List tasks'''
         from allura import model as M
@@ -194,6 +218,7 @@ class TaskCommand(base.Command):
             q = dict()
         else:
             q = dict(state=self.options.state)
+        q = self._add_filters(q)
         for t in M.MonQTask.query.find(q):
             print(t)
 
@@ -201,8 +226,10 @@ class TaskCommand(base.Command):
         '''Retry tasks in an error state'''
         from allura import model as M
         base.log.info('Retry tasks in error state')
+        q = dict(state='error')
+        q = self._add_filters(q)
         M.MonQTask.query.update(
-            dict(state='error'),
+            q,
             {'$set': dict(state='ready')},
             multi=True)
 
@@ -210,8 +237,9 @@ class TaskCommand(base.Command):
         '''Purge completed tasks'''
         from allura import model as M
         base.log.info('Purge complete/forget tasks')
-        M.MonQTask.query.remove(
-            dict(state='complete', result_type='forget'))
+        q = dict(state='complete', result_type='forget')
+        q = self._add_filters(q)
+        M.MonQTask.query.remove(q)
 
     def _timeout(self):
         '''Reset tasks that have been busy too long to 'ready' state'''
@@ -219,7 +247,15 @@ class TaskCommand(base.Command):
         base.log.info('Reset tasks stuck for %ss or more',
                       self.options.timeout)
         cutoff = datetime.utcnow() - timedelta(seconds=self.options.timeout)
-        M.MonQTask.timeout_tasks(cutoff)
+        q = dict(
+            state='busy',
+            time_start={'$lt': cutoff},
+        )
+        q = self._add_filters(q)
+        M.MonQTask.query.update(
+            q,
+            {'$set': dict(state='ready')},
+            multi=True)
 
     def _commit(self):
         '''Schedule a SOLR commit'''
diff --git a/Allura/allura/model/monq_model.py b/Allura/allura/model/monq_model.py
index 60edf10..2af31ed 100644
--- a/Allura/allura/model/monq_model.py
+++ b/Allura/allura/model/monq_model.py
@@ -217,20 +217,6 @@ class MonQTask(MappedClass):
                 return None
 
     @classmethod
-    def timeout_tasks(cls, older_than):
-        '''Mark all busy tasks older than a certain datetime as 'ready' again.
-        Used to retry 'stuck' tasks.'''
-        spec = dict(state='busy')
-        spec['time_start'] = {'$lt': older_than}
-        cls.query.update(spec, {'$set': dict(state='ready')}, multi=True)
-
-    @classmethod
-    def clear_complete(cls):
-        '''Delete the task objects for complete tasks'''
-        spec = dict(state='complete')
-        cls.query.remove(spec)
-
-    @classmethod
     def run_ready(cls, worker=None):
         '''Run all the tasks that are currently ready'''
         i = 0
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index ab077fd..ca6980f 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -280,6 +280,14 @@ class TestTaskCommand(object):
         exit_code = taskd.TaskCommand('task').run([test_config, 'list'])
         assert_equal(exit_code, 0)
 
+    def test_retry(self):
+        exit_code = taskd.TaskCommand('task').run([
+            test_config, 'retry',
+            '--filter-name-prefix', 'allura.tasks.index_tasks.',
+            '--filter-result-regex', 'pysolr',
+        ])
+        assert_equal(exit_code, 0)
+
 
 class TestTaskdCleanupCommand(object):
 


Mime
View raw message