allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From brond...@apache.org
Subject [03/18] git commit: ticket:562 Added session extension for indexed objects
Date Fri, 09 May 2014 21:42:23 GMT
ticket:562 Added session extension for indexed objects


Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/e5fa710e
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/e5fa710e
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/e5fa710e

Branch: refs/heads/db/7257
Commit: e5fa710edb2ee267c03b1e6c087cbb58dfa24b3c
Parents: 81cbc1d
Author: Ferens Dmitriy <ferensdima@gmail.com>
Authored: Thu Mar 27 23:02:14 2014 +0200
Committer: Dave Brondsema <dbrondsema@slashdotmedia.com>
Committed: Fri May 9 18:56:46 2014 +0000

----------------------------------------------------------------------
 Allura/allura/lib/search.py              |  3 --
 Allura/allura/model/session.py           | 34 ++++++++++++++----
 Allura/allura/tasks/index_tasks.py       | 50 ++++++++++++++++++---------
 Allura/allura/tests/test_tasks.py        | 19 +++++++---
 Allura/allura/tests/unit/test_session.py | 29 +++++++++++++++-
 5 files changed, 105 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/allura/blob/e5fa710e/Allura/allura/lib/search.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/search.py b/Allura/allura/lib/search.py
index 27cc05e..7751552 100644
--- a/Allura/allura/lib/search.py
+++ b/Allura/allura/lib/search.py
@@ -88,9 +88,6 @@ class SearchIndexable(object):
         doc['text'] = jinja2.Markup.escape(text).striptags()
         return doc
 
-    def add_to_solr(self, solr_instance):
-        solr_instance.add([self.solarize()])
-
 
 class SearchError(SolrError):
     pass

http://git-wip-us.apache.org/repos/asf/allura/blob/e5fa710e/Allura/allura/model/session.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/session.py b/Allura/allura/model/session.py
index ca5db5a..2b38440 100644
--- a/Allura/allura/model/session.py
+++ b/Allura/allura/model/session.py
@@ -29,7 +29,7 @@ from allura.tasks import index_tasks
 log = logging.getLogger(__name__)
 
 
-class ArtifactSessionExtension(SessionExtension):
+class ManagedSessionExtension(SessionExtension):
 
     def __init__(self, session):
         SessionExtension.__init__(self, session)
@@ -51,9 +51,32 @@ class ArtifactSessionExtension(SessionExtension):
             elif st.status == st.deleted:
                 self.objects_deleted = [obj]
 
+
+class IndexerSessionExtension(ManagedSessionExtension):
+    task_add = None
+    task_del = None
+
+    def after_flush(self, obj=None):
+        if (self.task_add or self.task_del) is None:
+            raise Exception('Both "task_add" and "task_del" must be defined')
+
+        modified = self.objects_added + self.objects_modified
+        if modified:
+            self.task_add.post([o._id for o in modified])
+        if self.objects_deleted:
+            self.task_del.post([o.index_id() for o in self.objects_deleted])
+
+
+class ProjectIndexerSessionExtension(IndexerSessionExtension):
+    task_add = index_tasks.add_projects
+    tasd_del = index_tasks.del_projects
+
+
+class ArtifactSessionExtension(ManagedSessionExtension):
+
     def after_flush(self, obj=None):
         "Update artifact references, and add/update this artifact to solr"
-        if not getattr(self.session, 'disable_artifact_index', False):
+        if not getattr(self.session, 'disable_index', False):
             from pylons import app_globals as g
             from .index import ArtifactReference, Shortlink
             from .session import main_orm_session
@@ -100,9 +123,6 @@ class BatchIndexer(ArtifactSessionExtension):
     to_delete = set()
     to_add = set()
 
-    def __init__(self, session):
-        ArtifactSessionExtension.__init__(self, session)
-
     def update_index(self, objects_deleted, arefs_added):
         """
         Caches adds and deletes for handling later.  Called after each flush of
@@ -197,7 +217,9 @@ main_doc_session = Session.by_name('main')
 project_doc_session = Session.by_name('project')
 task_doc_session = Session.by_name('task')
 main_orm_session = ThreadLocalORMSession(main_doc_session)
-project_orm_session = ThreadLocalORMSession(project_doc_session)
+project_orm_session = ThreadLocalORMSession(
+    doc_session=project_doc_session,
+    extensions=[ProjectIndexerSessionExtension])
 task_orm_session = ThreadLocalORMSession(task_doc_session)
 artifact_orm_session = ThreadLocalORMSession(
     doc_session=project_doc_session,

http://git-wip-us.apache.org/repos/asf/allura/blob/e5fa710e/Allura/allura/tasks/index_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/index_tasks.py b/Allura/allura/tasks/index_tasks.py
index bb54118..31f5dc7 100644
--- a/Allura/allura/tasks/index_tasks.py
+++ b/Allura/allura/tasks/index_tasks.py
@@ -29,19 +29,37 @@ from allura.lib.solr import make_solr_from_config
 log = logging.getLogger(__name__)
 
 
+class GenericIndexHandler(object):
+    _instance = None
+
+    def __new__(cls):
+        if not cls._instance:
+            cls._instance = super(GenericIndexHandler, cls).__new__(cls)
+        return cls._instance
+
+    def get_solr(self, solr_hosts=None):
+        return make_solr_from_config(solr_hosts) if solr_hosts else g.solr
+
+    def add_objects(self, objects, solr_hosts=None):
+        solr_instance = self.get_solr(solr_hosts)
+        solr_instance.add(obj.solarize() for obj in objects)
+
+    def del_objects(self, object_solr_ids):
+        solr_instance = self.get_solr()
+        solr_query = 'id:({0})'.format(' || '.join(object_solr_ids))
+        solr_instance.delete(q=solr_query)
+
+
 @task
-def add_project(project_id, solr_hosts=None):
-    '''
-    Add project to SOLR.
+def add_projects(project_ids):
+    from allura.model.project import Project
+    projects = Project.query.find(dict(_id={'$in': project_ids})).all()
+    GenericIndexHandler().add_objects(projects)
 
-    :param solr_hosts: a list of solr hists to use instead of defaults
-    :type solr_hosts: list of strings
-    '''
-    from allura import model as M
 
-    solr = make_solr_from_config(solr_hosts) if solr_hosts else g.solr
-    project = M.Project.query.get(_id=project_id)
-    project.add_to_solr(solr)
+@task
+def del_projects(project_solr_ids):
+    GenericIndexHandler().del_objects(project_solr_ids)
 
 
 @task
@@ -60,7 +78,7 @@ def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
     solr_updates = []
     with _indexing_disabled(M.session.artifact_orm_session._get()):
         for ref in M.ArtifactReference.query.find(dict(_id={'$in': ref_ids})):
-            # try:
+            try:
                 artifact = ref.artifact
                 s = artifact.solarize()
                 if s is None:
@@ -75,9 +93,9 @@ def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
                     link_text = artifact.index().get('text') or ''
                     shortlinks = find_shortlinks(link_text)
                     ref.references = [link.ref_id for link in shortlinks]
-            # except Exception:
-            #     log.error('Error indexing artifact %s', ref._id)
-            #     exceptions.append(sys.exc_info())
+            except Exception:
+                log.error('Error indexing artifact %s', ref._id)
+                exceptions.append(sys.exc_info())
         solr.add(solr_updates)
 
     if len(exceptions) == 1:
@@ -104,8 +122,8 @@ def commit():
 
 @contextmanager
 def _indexing_disabled(session):
-    session.disable_artifact_index = session.skip_mod_date = True
+    session.disable_index = session.skip_mod_date = True
     try:
         yield session
     finally:
-        session.disable_artifact_index = session.skip_mod_date = False
+        session.disable_index = session.skip_mod_date = False

http://git-wip-us.apache.org/repos/asf/allura/blob/e5fa710e/Allura/allura/tests/test_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_tasks.py b/Allura/allura/tests/test_tasks.py
index 01df2c4..04e03c2 100644
--- a/Allura/allura/tests/test_tasks.py
+++ b/Allura/allura/tests/test_tasks.py
@@ -101,12 +101,23 @@ class TestIndexTasks(unittest.TestCase):
         setup_basic_test()
         setup_global_objects()
 
-    def test_add_project(self):
+    def test_add_projects(self):
         old_solr_size = len(g.solr.db)
-        project = M.Project.query.find().first()
-        index_tasks.add_project(project._id)
+        projects = M.Project.query.find().all()
+        index_tasks.add_projects([p._id for p in projects])
         new_solr_size = len(g.solr.db)
-        assert old_solr_size + 1 == new_solr_size
+        assert old_solr_size + len(projects) == new_solr_size
+
+    @td.with_wiki
+    def test_del_projects(self):
+        projects = M.Project.query.find().all()
+        index_tasks.add_projects([p._id for p in projects])
+
+        with mock.patch('allura.tasks.index_tasks.g.solr') as solr:
+            index_tasks.del_projects([p.index_id() for p in projects])
+            assert solr.delete.call_count, 1
+            for project in projects:
+                assert project.index_id() in solr.delete.call_args[1]['q']
 
     @td.with_wiki
     def test_add_artifacts(self):

http://git-wip-us.apache.org/repos/asf/allura/blob/e5fa710e/Allura/allura/tests/unit/test_session.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/unit/test_session.py b/Allura/allura/tests/unit/test_session.py
index eec41bd..649dd4a 100644
--- a/Allura/allura/tests/unit/test_session.py
+++ b/Allura/allura/tests/unit/test_session.py
@@ -21,7 +21,8 @@ import mock
 from unittest import TestCase
 
 from allura.tests import decorators as td
-from allura.model.session import BatchIndexer, substitute_extensions
+from allura.model.session import IndexerSessionExtension, BatchIndexer, \
+                                 substitute_extensions
 
 
 def test_extensions_cm():
@@ -68,6 +69,32 @@ def test_extensions_cm_flush_raises():
     assert session._kwargs['extensions'] == []
 
 
+class TestIndexerSessionExtension(TestCase):
+
+    def setUp(self):
+        session = mock.Mock()
+        self.ExtensionClass = IndexerSessionExtension
+        self.extension = self.ExtensionClass(session)
+        self.extension.task_add = mock.Mock()
+        self.extension.task_del = mock.Mock()
+
+    def _mock_indexable(self, **kw):
+        m = mock.Mock(**kw)
+        m.index_id.return_value = id(m)
+        return m
+
+    def test_flush(self):
+        added = [self._mock_indexable(_id=i) for i in (1, 2, 3)]
+        modified = [self._mock_indexable(_id=i) for i in (4, 5)]
+        deleted = [self._mock_indexable(_id=i) for i in (6, 7)]
+        self.extension.objects_added = added
+        self.extension.objects_modified = modified
+        self.extension.objects_deleted = deleted
+        self.extension.after_flush()
+        self.extension.task_add.post.assert_called_once_with([1, 2, 3, 4, 5])
+        self.extension.task_del.post.assert_called_once_with(map(id, deleted))
+
+
 class TestBatchIndexer(TestCase):
 
     def setUp(self):


Mime
View raw message