allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject git commit: [#6801] Made last_commit_ids for Git threaded
Date Mon, 11 Nov 2013 23:14:11 GMT
Updated Branches:
  refs/heads/cj/6801 [created] 3445934f2


[#6801] Made last_commit_ids for Git threaded

Signed-off-by: Cory Johns <cjohns@slashdotmedia.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/3445934f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/3445934f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/3445934f

Branch: refs/heads/cj/6801
Commit: 3445934f2cdbad06e5f85bbbc4e326f941a3d499
Parents: f1c9240
Author: Cory Johns <cjohns@slashdotmedia.com>
Authored: Mon Nov 11 23:12:22 2013 +0000
Committer: Cory Johns <cjohns@slashdotmedia.com>
Committed: Mon Nov 11 23:12:22 2013 +0000

----------------------------------------------------------------------
 ForgeGit/forgegit/model/git_repo.py             | 61 +++++++++++++-------
 .../forgegit/tests/model/test_repository.py     |  4 ++
 2 files changed, 45 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3445934f/ForgeGit/forgegit/model/git_repo.py
----------------------------------------------------------------------
diff --git a/ForgeGit/forgegit/model/git_repo.py b/ForgeGit/forgegit/model/git_repo.py
index a06300f..56bd125 100644
--- a/ForgeGit/forgegit/model/git_repo.py
+++ b/ForgeGit/forgegit/model/git_repo.py
@@ -27,6 +27,8 @@ from datetime import datetime
 from glob import glob
 import gzip
 from time import time
+from threading import Thread
+from Queue import Queue
 
 import tg
 import git
@@ -34,7 +36,7 @@ import gitdb
 from pylons import app_globals as g
 from pylons import tmpl_context as c
 from pymongo.errors import DuplicateKeyError
-from paste.deploy.converters import asbool
+from paste.deploy.converters import asbool, asint
 
 from ming.base import Object
 from ming.orm import Mapper, session, mapper
@@ -502,27 +504,46 @@ class GitImplementation(M.RepositoryImplementation):
         """
         Find the ID of the last commit to touch each path.
         """
-        result = {}
-        paths = set(paths)
-        orig_commit_id = commit_id = commit._id
+        if not paths:
+            return {}
         timeout = float(tg.config.get('lcd_timeout', 60))
         start_time = time()
-        while paths and commit_id:
-            if time() - start_time > timeout:
-                log.error('last_commit_ids timeout for %s on %s', orig_commit_id, ', '.join(paths))
-                return result
-            lines = self._git.git.log(
-                    orig_commit_id, '--', *paths,
-                    pretty='format:%H',
-                    name_only=True,
-                    max_count=1,
-                    no_merges=True).split('\n')
-            commit_id = lines[0]
-            changes = set(lines[1:])
-            changed = prefix_paths_union(paths, changes)
-            for path in changed:
-                result[path] = commit_id
-            paths -= changed
+        paths = list(set(paths))  # remove dupes
+        result = {}  # will be appended to from each thread
+        chunks = Queue()
+        lcd_chunk_size = asint(tg.config.get('lcd_thread_chunk_size', 10))
+        num_threads = 0
+        for s in range(0, len(paths), lcd_chunk_size):
+            chunks.put(paths[s:s+lcd_chunk_size])
+            num_threads += 1
+        def get_ids():
+            paths = set(chunks.get())
+            commit_id = commit._id
+            while paths and commit_id:
+                if time() - start_time > timeout:
+                    log.error('last_commit_ids timeout for %s on %s', commit._id, ', '.join(paths))
+                    break
+                lines = self._git.git.log(
+                        commit._id, '--', *paths,
+                        pretty='format:%H',
+                        name_only=True,
+                        max_count=1,
+                        no_merges=True).split('\n')
+                commit_id = lines[0]
+                changes = set(lines[1:])
+                changed = prefix_paths_union(paths, changes)
+                for path in changed:
+                    result[path] = commit_id
+                paths -= changed
+            chunks.task_done()
+            return
+        if num_threads == 1:
+            get_ids()
+        else:
+            for i in range(num_threads):
+                t = Thread(target=get_ids)
+                t.start()
+            chunks.join()
         return result
 
 class _OpenedGitBlob(object):

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3445934f/ForgeGit/forgegit/tests/model/test_repository.py
----------------------------------------------------------------------
diff --git a/ForgeGit/forgegit/tests/model/test_repository.py b/ForgeGit/forgegit/tests/model/test_repository.py
index ea615b6..446b505 100644
--- a/ForgeGit/forgegit/tests/model/test_repository.py
+++ b/ForgeGit/forgegit/tests/model/test_repository.py
@@ -446,6 +446,10 @@ class TestGitImplementation(unittest.TestCase):
                 'f2.txt': '259c77dd6ee0e6091d11e429b56c44ccbf1e64a3',
             })
 
+    def test_last_commit_ids_threaded(self):
+        with h.push_config(tg.config, lcd_thread_chunk_size=1):
+            self.test_last_commit_ids()
+
 
 class TestGitCommit(unittest.TestCase):
 


Mime
View raw message