allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tvansteenbu...@apache.org
Subject git commit: [#6801] Made last_commit_ids for Git threaded
Date Tue, 19 Nov 2013 23:50:03 GMT
Updated Branches:
  refs/heads/master dcdb4ca77 -> 01cfb187b


[#6801] Made last_commit_ids for Git threaded

Signed-off-by: Cory Johns <cjohns@slashdotmedia.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/01cfb187
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/01cfb187
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/01cfb187

Branch: refs/heads/master
Commit: 01cfb187b9e80b11e06b89f850e98e9854c895ed
Parents: dcdb4ca
Author: Cory Johns <cjohns@slashdotmedia.com>
Authored: Mon Nov 11 23:12:22 2013 +0000
Committer: Tim Van Steenburgh <tvansteenburgh@gmail.com>
Committed: Tue Nov 19 23:49:13 2013 +0000

----------------------------------------------------------------------
 ForgeGit/forgegit/model/git_repo.py             | 61 +++++++++++++-------
 .../forgegit/tests/model/test_repository.py     |  4 ++
 2 files changed, 45 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/01cfb187/ForgeGit/forgegit/model/git_repo.py
----------------------------------------------------------------------
diff --git a/ForgeGit/forgegit/model/git_repo.py b/ForgeGit/forgegit/model/git_repo.py
index 516b6b1..145e528 100644
--- a/ForgeGit/forgegit/model/git_repo.py
+++ b/ForgeGit/forgegit/model/git_repo.py
@@ -27,6 +27,8 @@ from datetime import datetime
 from glob import glob
 import gzip
 from time import time
+from threading import Thread
+from Queue import Queue
 
 import tg
 import git
@@ -34,7 +36,7 @@ import gitdb
 from pylons import app_globals as g
 from pylons import tmpl_context as c
 from pymongo.errors import DuplicateKeyError
-from paste.deploy.converters import asbool
+from paste.deploy.converters import asbool, asint
 
 from ming.base import Object
 from ming.orm import Mapper, session, mapper
@@ -502,27 +504,46 @@ class GitImplementation(M.RepositoryImplementation):
         """
         Find the ID of the last commit to touch each path.
         """
-        result = {}
-        paths = set(paths)
-        orig_commit_id = commit_id = commit._id
+        if not paths:
+            return {}
         timeout = float(tg.config.get('lcd_timeout', 60))
         start_time = time()
-        while paths and commit_id:
-            if time() - start_time > timeout:
-                log.error('last_commit_ids timeout for %s on %s', orig_commit_id, ', '.join(paths))
-                return result
-            lines = self._git.git.log(
-                    orig_commit_id, '--', *paths,
-                    pretty='format:%H',
-                    name_only=True,
-                    max_count=1,
-                    no_merges=True).split('\n')
-            commit_id = lines[0]
-            changes = set(lines[1:])
-            changed = prefix_paths_union(paths, changes)
-            for path in changed:
-                result[path] = commit_id
-            paths -= changed
+        paths = list(set(paths))  # remove dupes
+        result = {}  # will be appended to from each thread
+        chunks = Queue()
+        lcd_chunk_size = asint(tg.config.get('lcd_thread_chunk_size', 10))
+        num_threads = 0
+        for s in range(0, len(paths), lcd_chunk_size):
+            chunks.put(paths[s:s+lcd_chunk_size])
+            num_threads += 1
+        def get_ids():
+            paths = set(chunks.get())
+            commit_id = commit._id
+            while paths and commit_id:
+                if time() - start_time > timeout:
+                    log.error('last_commit_ids timeout for %s on %s', commit._id, ', '.join(paths))
+                    break
+                lines = self._git.git.log(
+                        commit._id, '--', *paths,
+                        pretty='format:%H',
+                        name_only=True,
+                        max_count=1,
+                        no_merges=True).split('\n')
+                commit_id = lines[0]
+                changes = set(lines[1:])
+                changed = prefix_paths_union(paths, changes)
+                for path in changed:
+                    result[path] = commit_id
+                paths -= changed
+            chunks.task_done()
+            return
+        if num_threads == 1:
+            get_ids()
+        else:
+            for i in range(num_threads):
+                t = Thread(target=get_ids)
+                t.start()
+            chunks.join()
         return result
 
 class _OpenedGitBlob(object):

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/01cfb187/ForgeGit/forgegit/tests/model/test_repository.py
----------------------------------------------------------------------
diff --git a/ForgeGit/forgegit/tests/model/test_repository.py b/ForgeGit/forgegit/tests/model/test_repository.py
index dfcd3ac..d2a9413 100644
--- a/ForgeGit/forgegit/tests/model/test_repository.py
+++ b/ForgeGit/forgegit/tests/model/test_repository.py
@@ -452,6 +452,10 @@ class TestGitImplementation(unittest.TestCase):
                 'f2.txt': '259c77dd6ee0e6091d11e429b56c44ccbf1e64a3',
             })
 
+    def test_last_commit_ids_threaded(self):
+        with h.push_config(tg.config, lcd_thread_chunk_size=1):
+            self.test_last_commit_ids()
+
 
 class TestGitCommit(unittest.TestCase):
 


Mime
View raw message