mirrors/git
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

git-p4: use -m when running p4 changes

Browse Source 
Simply running "p4 changes" on a large branch can result in a "too
many rows scanned" error from the Perforce server. It is better to
use a sequence of smaller calls to "p4 changes", using the "-m"
option to limit the size of each call.

Signed-off-by: Lex Spoon <lex@lexspoon.org>
Acked-by: Luke Diamand <luke@diamand.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
This commit is contained in:
Lex Spoon
2015-04-20 11:00:20 -04:00
committed by Junio C Hamano
parent 3d6bc9a763
commit 96b2d54aee
3 changed files with 119 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
git-p4.py
View File

@ -740,17 +740,43 @@ def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent
def originP4BranchesExist():
return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
def p4ChangesForPaths(depotPaths, changeRange):
def p4ChangesForPaths(depotPaths, changeRange, block_size):
assert depotPaths
cmd = ['changes']
for p in depotPaths:
cmd += ["%s...%s" % (p, changeRange)]
output = p4_read_pipe_lines(cmd)
assert block_size
# Parse the change range into start and end
if changeRange is None or changeRange == '':
changeStart = '@1'
changeEnd = '#head'
else:
parts = changeRange.split(',')
assert len(parts) == 2
changeStart = parts[0]
changeEnd = parts[1]
# Accumulate change numbers in a dictionary to avoid duplicates
changes = {}
for line in output:
changeNum = int(line.split(" ")[1])
changes[changeNum] = True
for p in depotPaths:
# Retrieve changes a block at a time, to prevent running
# into a MaxScanRows error from the server.
start = changeStart
end = changeEnd
get_another_block = True
while get_another_block:
new_changes = []
cmd = ['changes']
cmd += ['-m', str(block_size)]
cmd += ["%s...%s,%s" % (p, start, end)]
for line in p4_read_pipe_lines(cmd):
changeNum = int(line.split(" ")[1])
new_changes.append(changeNum)
changes[changeNum] = True
if len(new_changes) == block_size:
get_another_block = True
end = '@' + str(min(new_changes))
else:
get_another_block = False
changelist = changes.keys()
changelist.sort()
@ -1911,7 +1937,10 @@ class P4Sync(Command, P4UserMap):
optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
help="Import into refs/heads/ , not refs/remotes"),
optparse.make_option("--max-changes", dest="maxChanges"),
optparse.make_option("--max-changes", dest="maxChanges",
help="Maximum number of changes to import"),
optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
help="Internal block size to use when iteratively calling p4 changes"),
optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
@ -1940,6 +1969,7 @@ class P4Sync(Command, P4UserMap):
self.syncWithOrigin = True
self.importIntoRemotes = True
self.maxChanges = ""
self.changes_block_size = 500
self.keepRepoPath = False
self.depotPaths = None
self.p4BranchesInGit = []
@ -2586,7 +2616,7 @@ class P4Sync(Command, P4UserMap):
branchPrefix = self.depotPaths[0] + branch + "/"
range = "@1,%s" % maxChange
#print "prefix" + branchPrefix
changes = p4ChangesForPaths([branchPrefix], range)
changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
if len(changes) <= 0:
return False
firstChange = changes[0]
@ -3002,7 +3032,7 @@ class P4Sync(Command, P4UserMap):
if self.verbose:
print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
self.changeRange)
changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
if len(self.maxChanges) > 0:
changes = changes[:min(int(self.maxChanges), len(changes))]