From fcd9bd854ad80637f711d7e984bf82320f4d3a6e Mon Sep 17 00:00:00 2001 From: Buba Date: Sat, 7 Feb 2026 19:37:22 +0000 Subject: [PATCH] Move tools to development repo --- .claude/commands/fix-lint-issue.md | 23 - .claude/commands/improve-docs.md | 60 - scripts/git-filter-repo.py | 4005 ---------------------------- 3 files changed, 4088 deletions(-) delete mode 100644 .claude/commands/fix-lint-issue.md delete mode 100644 .claude/commands/improve-docs.md delete mode 100755 scripts/git-filter-repo.py diff --git a/.claude/commands/fix-lint-issue.md b/.claude/commands/fix-lint-issue.md deleted file mode 100644 index 14542c73..00000000 --- a/.claude/commands/fix-lint-issue.md +++ /dev/null @@ -1,23 +0,0 @@ -# Fix specific lint issue (parameter: issue code or description) - -## Variables - -ISSUE_ID: $ARGUMENTS - -## Instructions - -1. Run "task lint-app" to identify all lint issues -2. Filter for issues matching "ISSUE_ID" (can be a specific error code like S607 or a description like "subprocess") -3. Analyze each matching issue and implement fixes according to best practices -4. For each fixed issue: - - Explain the problem and why it's a concern - - Show the original code - - Explain your fix and its benefits -5. After implementing fixes, run "task lint-app" again to verify the issues are resolved -6. Summarize your changes and list any remaining issues grouped by type - -Notes: -- Focus only on issues matching the specified ISSUE_ID -- Apply consistent fixes across similar issues -- For Python security issues (S-prefixed codes), consult Python security best practices -- For other issues, follow the appropriate linting tool's recommendations diff --git a/.claude/commands/improve-docs.md b/.claude/commands/improve-docs.md deleted file mode 100644 index 5d267fe1..00000000 --- a/.claude/commands/improve-docs.md +++ /dev/null @@ -1,60 +0,0 @@ -# Improve Documentation - -## Variables - -FILES: $ARGUMENTS - -## Instructions - -Improve the spelling, grammar, and style of markdown-based documentation files in this repository. - -### Part 1 - Identify Documentation Files - -1. If FILES is provided, use those specific files. Otherwise, find all markdown files in the repository using: - - Search for files with `.md` extension - - Focus on documentation directories like `docs/`, `README.md`, and other markdown files - - Exclude generated files, third-party code, and non-documentation markdown - -### Part 2 - Analyze and Improve Each File - -For each documentation file: - -1. **Read the file** to understand its content and purpose -2. **Check for issues**: - - Spelling errors - - Grammar mistakes - - Inconsistent formatting - - Awkward phrasing or unclear sentences - - Missing or incorrect punctuation - - Inconsistent heading styles - - Broken links or references -3. **Apply fixes**: - - Correct spelling errors - - Improve grammar and sentence structure - - Enhance clarity and readability - - Ensure consistent markdown formatting - - Fix any broken internal links - - Maintain the original meaning and technical accuracy - -### Part 3 - Quality Assurance - -1. After improving each file, re-read it to ensure: - - The content remains technically accurate - - The tone and style are appropriate for documentation - - Links and references still work - - Code examples (if any) are still valid -2. Run any available linting tools on the modified files (e.g., markdownlint if configured) -3. Verify that the changes don't break any documentation build processes - -### Part 4 - Summary - -Provide a summary of all changes made, including: -- Files modified -- Types of improvements made (spelling, grammar, style) -- Any files that couldn't be improved and why - -Notes: -- Preserve technical terminology and code snippets exactly as they are -- Maintain the existing documentation structure and organization -- Focus on clarity and professionalism while keeping the original voice -- If unsure about a technical term or concept, leave it unchanged diff --git a/scripts/git-filter-repo.py b/scripts/git-filter-repo.py deleted file mode 100755 index ae71fad1..00000000 --- a/scripts/git-filter-repo.py +++ /dev/null @@ -1,4005 +0,0 @@ -#!/usr/bin/env python3 - -""" -git-filter-repo filters git repositories, similar to git filter-branch, BFG -repo cleaner, and others. The basic idea is that it works by running - git fast-export | filter | git fast-import -where this program not only launches the whole pipeline but also serves as -the 'filter' in the middle. It does a few additional things on top as well -in order to make it into a well-rounded filtering tool. - -git-filter-repo can also be used as a library for more involved filtering -operations; however: - ***** API BACKWARD COMPATIBILITY CAVEAT ***** - Programs using git-filter-repo as a library can reach pretty far into its - internals, but I am not prepared to guarantee backward compatibility of - all APIs. I suspect changes will be rare, but I reserve the right to - change any API. Since it is assumed that repository filtering is - something one would do very rarely, and in particular that it's a - one-shot operation, this should not be a problem in practice for anyone. - However, if you want to re-use a program you have written that uses - git-filter-repo as a library (or makes use of one of its --*-callback - arguments), you should either make sure you are using the same version of - git and git-filter-repo, or make sure to re-test it. - - If there are particular pieces of the API you are concerned about, and - there is not already a testcase for it in t9391-lib-usage.sh or - t9392-python-callback.sh, please contribute a testcase. That will not - prevent me from changing the API, but it will allow you to look at the - history of a testcase to see whether and how the API changed. - ***** END API BACKWARD COMPATIBILITY CAVEAT ***** -""" - -import argparse -import collections -import fnmatch -import gettext -import io -import os -import platform -import re -import shutil -import subprocess -import sys -import time -import textwrap - -from datetime import tzinfo, timedelta, datetime - -__all__ = ["Blob", "Reset", "FileChange", "Commit", "Tag", "Progress", - "Checkpoint", "FastExportParser", "ProgressWriter", - "string_to_date", "date_to_string", - "record_id_rename", "GitUtils", "FilteringOptions", "RepoFilter"] - -deleted_hash = b'0'*40 -write_marks = True -date_format_permissive = True - -def gettext_poison(msg): - if "GIT_TEST_GETTEXT_POISON" in os.environ: # pragma: no cover - return "# GETTEXT POISON #" - return gettext.gettext(msg) - -_ = gettext_poison - -def setup_gettext(): - TEXTDOMAIN="git-filter-repo" - podir = os.environ.get("GIT_TEXTDOMAINDIR") or "@@LOCALEDIR@@" - if not os.path.isdir(podir): # pragma: no cover - podir = None # Python has its own fallback; use that - - ## This looks like the most straightforward translation of the relevant - ## code in git.git:gettext.c and git.git:perl/Git/I18n.pm: - #import locale - #locale.setlocale(locale.LC_MESSAGES, ""); - #locale.setlocale(locale.LC_TIME, ""); - #locale.textdomain(TEXTDOMAIN); - #locale.bindtextdomain(TEXTDOMAIN, podir); - ## but the python docs suggest using the gettext module (which doesn't - ## have setlocale()) instead, so: - gettext.textdomain(TEXTDOMAIN); - gettext.bindtextdomain(TEXTDOMAIN, podir); - -def _timedelta_to_seconds(delta): - """ - Converts timedelta to seconds - """ - offset = delta.days*86400 + delta.seconds + (delta.microseconds+0.0)/1000000 - return round(offset) - -class FixedTimeZone(tzinfo): - """ - Fixed offset in minutes east from UTC. - """ - - tz_re = re.compile(br'^([-+]?)(\d\d)(\d\d)$') - - def __init__(self, offset_string): - tzinfo.__init__(self) - sign, hh, mm = FixedTimeZone.tz_re.match(offset_string).groups() - factor = -1 if (sign and sign == b'-') else 1 - self._offset = timedelta(minutes = factor*(60*int(hh) + int(mm))) - self._offset_string = offset_string - - def utcoffset(self, dt): - return self._offset - - def tzname(self, dt): - return self._offset_string - - def dst(self, dt): - return timedelta(0) - -def string_to_date(datestring): - (unix_timestamp, tz_offset) = datestring.split() - return datetime.fromtimestamp(int(unix_timestamp), - FixedTimeZone(tz_offset)) - -def date_to_string(dateobj): - epoch = datetime.fromtimestamp(0, dateobj.tzinfo) - return(b'%d %s' % (int(_timedelta_to_seconds(dateobj - epoch)), - dateobj.tzinfo.tzname(0))) - -def decode(bytestr): - 'Try to convert bytestr to utf-8 for outputting as an error message.' - return bytestr.decode('utf-8', 'backslashreplace') - -def glob_to_regex(glob_bytestr): - 'Translate glob_bytestr into a regex on bytestrings' - - # fnmatch.translate is idiotic and won't accept bytestrings - if (decode(glob_bytestr).encode() != glob_bytestr): # pragma: no cover - raise SystemExit(_("Error: Cannot handle glob %s").format(glob_bytestr)) - - # Create regex operating on string - regex = fnmatch.translate(decode(glob_bytestr)) - - # FIXME: This is an ugly hack... - # fnmatch.translate tries to do multi-line matching and wants the glob to - # match up to the end of the input, which isn't relevant for us, so we - # have to modify the regex. fnmatch.translate has used different regex - # constructs to achieve this with different python versions, so we have - # to check for each of them and then fix it up. It would be much better - # if fnmatch.translate could just take some flags to allow us to specify - # what we want rather than employing this hackery, but since it - # doesn't... - if regex.endswith(r'\Z(?ms)'): # pragma: no cover - regex = regex[0:-7] - elif regex.startswith(r'(?s:') and regex.endswith(r')\Z'): # pragma: no cover - regex = regex[4:-3] - - # Finally, convert back to regex operating on bytestr - return regex.encode() - -class PathQuoting: - _unescape = {b'a': b'\a', - b'b': b'\b', - b'f': b'\f', - b'n': b'\n', - b'r': b'\r', - b't': b'\t', - b'v': b'\v', - b'"': b'"', - b'\\':b'\\'} - _unescape_re = re.compile(br'\\([a-z"\\]|[0-9]{3})') - _escape = [bytes([x]) for x in range(127)]+[ - b'\\'+bytes(ord(c) for c in oct(x)[2:]) for x in range(127,256)] - _reverse = dict(map(reversed, _unescape.items())) - for x in _reverse: - _escape[ord(x)] = b'\\'+_reverse[x] - _special_chars = [len(x) > 1 for x in _escape] - - @staticmethod - def unescape_sequence(orig): - seq = orig.group(1) - return PathQuoting._unescape[seq] if len(seq) == 1 else bytes([int(seq, 8)]) - - @staticmethod - def dequote(quoted_string): - if quoted_string.startswith(b'"'): - assert quoted_string.endswith(b'"') - return PathQuoting._unescape_re.sub(PathQuoting.unescape_sequence, - quoted_string[1:-1]) - return quoted_string - - @staticmethod - def enquote(unquoted_string): - # Option 1: Quoting when fast-export would: - # pqsc = PathQuoting._special_chars - # if any(pqsc[x] for x in set(unquoted_string)): - # Option 2, perf hack: do minimal amount of quoting required by fast-import - if unquoted_string.startswith(b'"') or b'\n' in unquoted_string: - pqe = PathQuoting._escape - return b'"' + b''.join(pqe[x] for x in unquoted_string) + b'"' - return unquoted_string - -class AncestryGraph(object): - """ - A class that maintains a direct acycle graph of commits for the purpose of - determining if one commit is the ancestor of another. - """ - - def __init__(self): - self.cur_value = 0 - - # A mapping from the external identifers given to us to the simple integers - # we use in self.graph - self.value = {} - - # A tuple of (depth, list-of-ancestors). Values and keys in this graph are - # all integers from the self.value dict. The depth of a commit is one more - # than the max depth of any of its ancestors. - self.graph = {} - - # Cached results from previous calls to is_ancestor(). - self._cached_is_ancestor = {} - - def record_external_commits(self, external_commits): - """ - Record in graph that each commit in external_commits exists, and is - treated as a root commit with no parents. - """ - for c in external_commits: - if c not in self.value: - self.cur_value += 1 - self.value[c] = self.cur_value - self.graph[self.cur_value] = (1, []) - - def add_commit_and_parents(self, commit, parents): - """ - Record in graph that commit has the given parents. parents _MUST_ have - been first recorded. commit _MUST_ not have been recorded yet. - """ - assert all(p in self.value for p in parents) - assert commit not in self.value - - # Get values for commit and parents - self.cur_value += 1 - self.value[commit] = self.cur_value - graph_parents = [self.value[x] for x in parents] - - # Determine depth for commit, then insert the info into the graph - depth = 1 - if parents: - depth += max(self.graph[p][0] for p in graph_parents) - self.graph[self.cur_value] = (depth, graph_parents) - - def is_ancestor(self, possible_ancestor, check): - """ - Return whether possible_ancestor is an ancestor of check - """ - a, b = self.value[possible_ancestor], self.value[check] - original_pair = (a,b) - a_depth = self.graph[a][0] - ancestors = [b] - visited = set() - while ancestors: - ancestor = ancestors.pop() - prev_pair = (a, ancestor) - if prev_pair in self._cached_is_ancestor: - if not self._cached_is_ancestor[prev_pair]: - continue - self._cached_is_ancestor[original_pair] = True - return True - if ancestor in visited: - continue - visited.add(ancestor) - depth, more_ancestors = self.graph[ancestor] - if ancestor == a: - self._cached_is_ancestor[original_pair] = True - return True - elif depth <= a_depth: - continue - ancestors.extend(more_ancestors) - self._cached_is_ancestor[original_pair] = False - return False - -class MailmapInfo(object): - def __init__(self, filename): - self.changes = {} - self._parse_file(filename) - - def _parse_file(self, filename): - name_and_email_re = re.compile(br'(.*?)\s*<([^>]*)>\s*') - comment_re = re.compile(br'\s*#.*') - if not os.access(filename, os.R_OK): - raise SystemExit(_("Cannot read %s") % decode(filename)) - with open(filename, 'br') as f: - count = 0 - for line in f: - count += 1 - err = "Unparseable mailmap file: line #{} is bad: {}".format(count, line) - # Remove comments - line = comment_re.sub(b'', line) - # Remove leading and trailing whitespace - line = line.strip() - if not line: - continue - - m = name_and_email_re.match(line) - if not m: - raise SystemExit(err) - proper_name, proper_email = m.groups() - if len(line) == m.end(): - self.changes[(None, proper_email)] = (proper_name, proper_email) - continue - rest = line[m.end():] - m = name_and_email_re.match(rest) - if m: - commit_name, commit_email = m.groups() - if len(rest) != m.end(): - raise SystemExit(err) - else: - commit_name, commit_email = rest, None - self.changes[(commit_name, commit_email)] = (proper_name, proper_email) - - def translate(self, name, email): - ''' Given a name and email, return the expected new name and email from the - mailmap if there is a translation rule for it, otherwise just return - the given name and email.''' - for old, new in self.changes.items(): - old_name, old_email = old - new_name, new_email = new - if (old_email is None or email.lower() == old_email.lower()) and ( - name == old_name or not old_name): - return (new_name or name, new_email or email) - return (name, email) - -class ProgressWriter(object): - def __init__(self): - self._last_progress_update = time.time() - self._last_message = None - - def show(self, msg): - self._last_message = msg - now = time.time() - if now - self._last_progress_update > .1: - self._last_progress_update = now - sys.stdout.write("\r{}".format(msg)) - sys.stdout.flush() - - def finish(self): - self._last_progress_update = 0 - if self._last_message: - self.show(self._last_message) - sys.stdout.write("\n") - -class _IDs(object): - """ - A class that maintains the 'name domain' of all the 'marks' (short int - id for a blob/commit git object). The reason this mechanism is necessary - is because the text of fast-export may refer to an object using a different - mark than the mark that was assigned to that object using IDS.new(). This - class allows you to translate the fast-export marks (old) to the marks - assigned from IDS.new() (new). - - Note that there are two reasons why the marks may differ: (1) The - user manually creates Blob or Commit objects (for insertion into the - stream) (2) We're reading the data from two different repositories - and trying to combine the data (git fast-export will number ids from - 1...n, and having two 1's, two 2's, two 3's, causes issues). - """ - - def __init__(self): - """ - Init - """ - # The id for the next created blob/commit object - self._next_id = 1 - - # A map of old-ids to new-ids (1:1 map) - self._translation = {} - - # A map of new-ids to every old-id that points to the new-id (1:N map) - self._reverse_translation = {} - - def has_renames(self): - """ - Return whether there have been ids remapped to new values - """ - return bool(self._translation) - - def new(self): - """ - Should be called whenever a new blob or commit object is created. The - returned value should be used as the id/mark for that object. - """ - rv = self._next_id - self._next_id += 1 - return rv - - def record_rename(self, old_id, new_id, handle_transitivity = False): - """ - Record that old_id is being renamed to new_id. - """ - if old_id != new_id: - # old_id -> new_id - self._translation[old_id] = new_id - - # Transitivity will be needed if new commits are being inserted mid-way - # through a branch. - if handle_transitivity: - # Anything that points to old_id should point to new_id - if old_id in self._reverse_translation: - for id_ in self._reverse_translation[old_id]: - self._translation[id_] = new_id - - # Record that new_id is pointed to by old_id - if new_id not in self._reverse_translation: - self._reverse_translation[new_id] = [] - self._reverse_translation[new_id].append(old_id) - - def translate(self, old_id): - """ - If old_id has been mapped to an alternate id, return the alternate id. - """ - if old_id in self._translation: - return self._translation[old_id] - else: - return old_id - - def __str__(self): - """ - Convert IDs to string; used for debugging - """ - rv = "Current count: %d\nTranslation:\n" % self._next_id - for k in sorted(self._translation): - rv += " %d -> %s\n" % (k, self._translation[k]) - - rv += "Reverse translation:\n" - for k in sorted(self._reverse_translation): - rv += " " + str(k) + " -> " + str(self._reverse_translation[k]) + "\n" - - return rv - -class _GitElement(object): - """ - The base class for all git elements that we create. - """ - - def __init__(self): - # A string that describes what type of Git element this is - self.type = None - - # A flag telling us if this Git element has been dumped - # (i.e. printed) or skipped. Typically elements that have been - # dumped or skipped will not be dumped again. - self.dumped = 0 - - def dump(self, file_): - """ - This version should never be called. Derived classes need to - override! We should note that subclasses should implement this - method such that the output would match the format produced by - fast-export. - """ - raise SystemExit(_("Unimplemented function: %s") % type(self).__name__ - +".dump()") # pragma: no cover - - def __bytes__(self): - """ - Convert GitElement to bytestring; used for debugging - """ - old_dumped = self.dumped - writeme = io.BytesIO() - self.dump(writeme) - output_lines = writeme.getvalue().splitlines() - writeme.close() - self.dumped = old_dumped - return b"%s:\n %s" % (type(self).__name__.encode(), - b"\n ".join(output_lines)) - - def skip(self, new_id=None): - """ - Ensures this element will not be written to output - """ - self.dumped = 2 - -class _GitElementWithId(_GitElement): - """ - The base class for Git elements that have IDs (commits and blobs) - """ - - def __init__(self): - _GitElement.__init__(self) - - # The mark (short, portable id) for this element - self.id = _IDS.new() - - # The previous mark for this element - self.old_id = None - - def skip(self, new_id=None): - """ - This element will no longer be automatically written to output. When a - commit gets skipped, it's ID will need to be translated to that of its - parent. - """ - self.dumped = 2 - - _IDS.record_rename(self.old_id or self.id, new_id) - -class Blob(_GitElementWithId): - """ - This class defines our representation of git blob elements (i.e. our - way of representing file contents). - """ - - def __init__(self, data, original_id = None): - _GitElementWithId.__init__(self) - - # Denote that this is a blob - self.type = 'blob' - - # Record original id - self.original_id = original_id - - # Stores the blob's data - assert(type(data) == bytes) - self.data = data - - def dump(self, file_): - """ - Write this blob element to a file. - """ - self.dumped = 1 - HASH_TO_ID[self.original_id] = self.id - ID_TO_HASH[self.id] = self.original_id - - file_.write(b'blob\n') - file_.write(b'mark :%d\n' % self.id) - file_.write(b'data %d\n%s' % (len(self.data), self.data)) - file_.write(b'\n') - - -class Reset(_GitElement): - """ - This class defines our representation of git reset elements. A reset - event is the creation (or recreation) of a named branch, optionally - starting from a specific revision). - """ - - def __init__(self, ref, from_ref = None): - _GitElement.__init__(self) - - # Denote that this is a reset - self.type = 'reset' - - # The name of the branch being (re)created - self.ref = ref - - # Some reference to the branch/commit we are resetting from - self.from_ref = from_ref - - def dump(self, file_): - """ - Write this reset element to a file - """ - self.dumped = 1 - - file_.write(b'reset %s\n' % self.ref) - if self.from_ref: - if isinstance(self.from_ref, int): - file_.write(b'from :%d\n' % self.from_ref) - else: - file_.write(b'from %s\n' % self.from_ref) - file_.write(b'\n') - -class FileChange(_GitElement): - """ - This class defines our representation of file change elements. File change - elements are components within a Commit element. - """ - - def __init__(self, type_, filename = None, id_ = None, mode = None): - _GitElement.__init__(self) - - # Denote the type of file-change (b'M' for modify, b'D' for delete, etc) - # We could - # assert(type(type_) == bytes) - # here but I don't just due to worries about performance overhead... - self.type = type_ - - # Record the name of the file being changed - self.filename = filename - - # Record the mode (mode describes type of file entry (non-executable, - # executable, or symlink)). - self.mode = mode - - # blob_id is the id (mark) of the affected blob - self.blob_id = id_ - - if type_ == b'DELETEALL': - assert filename is None and id_ is None and mode is None - self.filename = b'' # Just so PathQuoting.enquote doesn't die - else: - assert filename is not None - - if type_ == b'M': - assert id_ is not None and mode is not None - elif type_ == b'D': - assert id_ is None and mode is None - elif type_ == b'R': # pragma: no cover (now avoid fast-export renames) - assert mode is None - if id_ is None: - raise SystemExit(_("new name needed for rename of %s") % filename) - self.filename = (self.filename, id_) - self.blob_id = None - - def dump(self, file_): - """ - Write this file-change element to a file - """ - skipped_blob = (self.type == b'M' and self.blob_id is None) - if skipped_blob: return - self.dumped = 1 - - quoted_filename = PathQuoting.enquote(self.filename) - if self.type == b'M' and isinstance(self.blob_id, int): - file_.write(b'M %s :%d %s\n' % (self.mode, self.blob_id, quoted_filename)) - elif self.type == b'M': - file_.write(b'M %s %s %s\n' % (self.mode, self.blob_id, quoted_filename)) - elif self.type == b'D': - file_.write(b'D %s\n' % quoted_filename) - elif self.type == b'DELETEALL': - file_.write(b'deleteall\n') - else: - raise SystemExit(_("Unhandled filechange type: %s") % self.type) # pragma: no cover - -class Commit(_GitElementWithId): - """ - This class defines our representation of commit elements. Commit elements - contain all the information associated with a commit. - """ - - def __init__(self, branch, - author_name, author_email, author_date, - committer_name, committer_email, committer_date, - message, - file_changes, - parents, - original_id = None, - encoding = None, # encoding for message; None implies UTF-8 - **kwargs): - _GitElementWithId.__init__(self) - self.old_id = self.id - - # Denote that this is a commit element - self.type = 'commit' - - # Record the affected branch - self.branch = branch - - # Record original id - self.original_id = original_id - - # Record author's name - self.author_name = author_name - - # Record author's email - self.author_email = author_email - - # Record date of authoring - self.author_date = author_date - - # Record committer's name - self.committer_name = committer_name - - # Record committer's email - self.committer_email = committer_email - - # Record date the commit was made - self.committer_date = committer_date - - # Record commit message and its encoding - self.encoding = encoding - self.message = message - - # List of file-changes associated with this commit. Note that file-changes - # are also represented as git elements - self.file_changes = file_changes - - self.parents = parents - - def dump(self, file_): - """ - Write this commit element to a file. - """ - self.dumped = 1 - HASH_TO_ID[self.original_id] = self.id - ID_TO_HASH[self.id] = self.original_id - - # Make output to fast-import slightly easier for humans to read if the - # message has no trailing newline of its own; cosmetic, but a nice touch... - extra_newline = b'\n' - if self.message.endswith(b'\n') or not (self.parents or self.file_changes): - extra_newline = b'' - - if not self.parents: - file_.write(b'reset %s\n' % self.branch) - file_.write((b'commit %s\n' - b'mark :%d\n' - b'author %s <%s> %s\n' - b'committer %s <%s> %s\n' - ) % ( - self.branch, self.id, - self.author_name, self.author_email, self.author_date, - self.committer_name, self.committer_email, self.committer_date - )) - if self.encoding: - file_.write(b'encoding %s\n' % self.encoding) - file_.write(b'data %d\n%s%s' % - (len(self.message), self.message, extra_newline)) - for i, parent in enumerate(self.parents): - file_.write(b'from ' if i==0 else b'merge ') - if isinstance(parent, int): - file_.write(b':%d\n' % parent) - else: - file_.write(b'%s\n' % parent) - for change in self.file_changes: - change.dump(file_) - if not self.parents and not self.file_changes: - # Workaround a bug in pre-git-2.22 versions of fast-import with - # the get-mark directive. - file_.write(b'\n') - file_.write(b'\n') - - def first_parent(self): - """ - Return first parent commit - """ - if self.parents: - return self.parents[0] - return None - - def skip(self, new_id=None): - _SKIPPED_COMMITS.add(self.old_id or self.id) - _GitElementWithId.skip(self, new_id) - -class Tag(_GitElementWithId): - """ - This class defines our representation of annotated tag elements. - """ - - def __init__(self, ref, from_ref, - tagger_name, tagger_email, tagger_date, tag_msg, - original_id = None): - _GitElementWithId.__init__(self) - self.old_id = self.id - - # Denote that this is a tag element - self.type = 'tag' - - # Store the name of the tag - self.ref = ref - - # Store the entity being tagged (this should be a commit) - self.from_ref = from_ref - - # Record original id - self.original_id = original_id - - # Store the name of the tagger - self.tagger_name = tagger_name - - # Store the email of the tagger - self.tagger_email = tagger_email - - # Store the date - self.tagger_date = tagger_date - - # Store the tag message - self.message = tag_msg - - def dump(self, file_): - """ - Write this tag element to a file - """ - - self.dumped = 1 - HASH_TO_ID[self.original_id] = self.id - ID_TO_HASH[self.id] = self.original_id - - file_.write(b'tag %s\n' % self.ref) - if (write_marks and self.id): - file_.write(b'mark :%d\n' % self.id) - markfmt = b'from :%d\n' if isinstance(self.from_ref, int) else b'from %s\n' - file_.write(markfmt % self.from_ref) - if self.tagger_name: - file_.write(b'tagger %s <%s> ' % (self.tagger_name, self.tagger_email)) - file_.write(self.tagger_date) - file_.write(b'\n') - file_.write(b'data %d\n%s' % (len(self.message), self.message)) - file_.write(b'\n') - -class Progress(_GitElement): - """ - This class defines our representation of progress elements. The progress - element only contains a progress message, which is printed by fast-import - when it processes the progress output. - """ - - def __init__(self, message): - _GitElement.__init__(self) - - # Denote that this is a progress element - self.type = 'progress' - - # Store the progress message - self.message = message - - def dump(self, file_): - """ - Write this progress element to a file - """ - self.dumped = 1 - - file_.write(b'progress %s\n' % self.message) - file_.write(b'\n') - -class Checkpoint(_GitElement): - """ - This class defines our representation of checkpoint elements. These - elements represent events which force fast-import to close the current - packfile, start a new one, and to save out all current branch refs, tags - and marks. - """ - - def __init__(self): - _GitElement.__init__(self) - - # Denote that this is a checkpoint element - self.type = 'checkpoint' - - def dump(self, file_): - """ - Write this checkpoint element to a file - """ - self.dumped = 1 - - file_.write(b'checkpoint\n') - file_.write(b'\n') - -class LiteralCommand(_GitElement): - """ - This class defines our representation of commands. The literal command - includes only a single line, and is not processed in any special way. - """ - - def __init__(self, line): - _GitElement.__init__(self) - - # Denote that this is a literal element - self.type = 'literal' - - # Store the command - self.line = line - - def dump(self, file_): - """ - Write this progress element to a file - """ - self.dumped = 1 - - file_.write(self.line) - -class Alias(_GitElement): - """ - This class defines our representation of fast-import alias elements. An - alias element is the setting of one mark to the same sha1sum as another, - usually because the newer mark corresponded to a pruned commit. - """ - - def __init__(self, ref, to_ref): - _GitElement.__init__(self) - # Denote that this is a reset - self.type = 'alias' - - self.ref = ref - self.to_ref = to_ref - - def dump(self, file_): - """ - Write this reset element to a file - """ - self.dumped = 1 - - file_.write(b'alias\nmark :%d\nto :%d\n\n' % (self.ref, self.to_ref)) - -class FastExportParser(object): - """ - A class for parsing and handling the output from fast-export. This - class allows the user to register callbacks when various types of - data are encountered in the fast-export output. The basic idea is that, - FastExportParser takes fast-export output, creates the various objects - as it encounters them, the user gets to use/modify these objects via - callbacks, and finally FastExportParser outputs the modified objects - in fast-import format (presumably so they can be used to create a new - repo). - """ - - def __init__(self, - tag_callback = None, commit_callback = None, - blob_callback = None, progress_callback = None, - reset_callback = None, checkpoint_callback = None, - done_callback = None): - # Members below simply store callback functions for the various git - # elements - self._tag_callback = tag_callback - self._blob_callback = blob_callback - self._reset_callback = reset_callback - self._commit_callback = commit_callback - self._progress_callback = progress_callback - self._checkpoint_callback = checkpoint_callback - self._done_callback = done_callback - - # Keep track of which refs appear from the export, and which make it to - # the import (pruning of empty commits, renaming of refs, and creating - # new manual objects and inserting them can cause these to differ). - self._exported_refs = set() - self._imported_refs = set() - - # A list of the branches we've seen, plus the last known commit they - # pointed to. An entry in latest_*commit will be deleted if we get a - # reset for that branch. These are used because of fast-import's weird - # decision to allow having an implicit parent via naming the branch - # instead of requiring branches to be specified via 'from' directives. - self._latest_commit = {} - self._latest_orig_commit = {} - - # A handle to the input source for the fast-export data - self._input = None - - # A handle to the output file for the output we generate (we call dump - # on many of the git elements we create). - self._output = None - - # Stores the contents of the current line of input being parsed - self._currentline = '' - - # Compile some regexes and cache those - self._mark_re = re.compile(br'mark :(\d+)\n$') - self._parent_regexes = {} - parent_regex_rules = (br' :(\d+)\n$', br' ([0-9a-f]{40})\n') - for parent_refname in (b'from', b'merge'): - ans = [re.compile(parent_refname+x) for x in parent_regex_rules] - self._parent_regexes[parent_refname] = ans - self._quoted_string_re = re.compile(br'"(?:[^"\\]|\\.)*"') - self._refline_regexes = {} - for refline_name in (b'reset', b'commit', b'tag', b'progress'): - self._refline_regexes[refline_name] = re.compile(refline_name+b' (.*)\n$') - self._user_regexes = {} - for user in (b'author', b'committer', b'tagger'): - self._user_regexes[user] = re.compile(user + b' (.*?) <(.*?)> (.*)\n$') - - def _advance_currentline(self): - """ - Grab the next line of input - """ - self._currentline = self._input.readline() - - def _parse_optional_mark(self): - """ - If the current line contains a mark, parse it and advance to the - next line; return None otherwise - """ - mark = None - matches = self._mark_re.match(self._currentline) - if matches: - mark = int(matches.group(1)) - self._advance_currentline() - return mark - - def _parse_optional_parent_ref(self, refname): - """ - If the current line contains a reference to a parent commit, then - parse it and advance the current line; otherwise return None. Note - that the name of the reference ('from', 'merge') must match the - refname arg. - """ - orig_baseref, baseref = None, None - rule, altrule = self._parent_regexes[refname] - matches = rule.match(self._currentline) - if matches: - orig_baseref = int(matches.group(1)) - # We translate the parent commit mark to what it needs to be in - # our mark namespace - baseref = _IDS.translate(orig_baseref) - self._advance_currentline() - else: - matches = altrule.match(self._currentline) - if matches: - orig_baseref = matches.group(1) - baseref = orig_baseref - self._advance_currentline() - return orig_baseref, baseref - - def _parse_optional_filechange(self): - """ - If the current line contains a file-change object, then parse it - and advance the current line; otherwise return None. We only care - about file changes of type b'M' and b'D' (these are the only types - of file-changes that fast-export will provide). - """ - filechange = None - changetype = self._currentline[0:1] - if changetype == b'M': - (changetype, mode, idnum, path) = self._currentline.split(None, 3) - if idnum[0:1] == b':': - idnum = idnum[1:] - path = path.rstrip(b'\n') - # We translate the idnum to our id system - if len(idnum) != 40: - idnum = _IDS.translate( int(idnum) ) - if idnum is not None: - if path.startswith(b'"'): - path = PathQuoting.dequote(path) - filechange = FileChange(b'M', path, idnum, mode) - else: - filechange = b'skipped' - self._advance_currentline() - elif changetype == b'D': - (changetype, path) = self._currentline.split(None, 1) - path = path.rstrip(b'\n') - if path.startswith(b'"'): - path = PathQuoting.dequote(path) - filechange = FileChange(b'D', path) - self._advance_currentline() - elif changetype == b'R': # pragma: no cover (now avoid fast-export renames) - rest = self._currentline[2:-1] - if rest.startswith(b'"'): - m = self._quoted_string_re.match(rest) - if not m: - raise SystemExit(_("Couldn't parse rename source")) - orig = PathQuoting.dequote(m.group(0)) - new = rest[m.end()+1:] - else: - orig, new = rest.split(b' ', 1) - if new.startswith(b'"'): - new = PathQuoting.dequote(new) - filechange = FileChange(b'R', orig, new) - self._advance_currentline() - return filechange - - def _parse_original_id(self): - original_id = self._currentline[len(b'original-oid '):].rstrip() - self._advance_currentline() - return original_id - - def _parse_encoding(self): - encoding = self._currentline[len(b'encoding '):].rstrip() - self._advance_currentline() - return encoding - - def _parse_ref_line(self, refname): - """ - Parses string data (often a branch name) from current-line. The name of - the string data must match the refname arg. The program will crash if - current-line does not match, so current-line will always be advanced if - this method returns. - """ - matches = self._refline_regexes[refname].match(self._currentline) - if not matches: - raise SystemExit(_("Malformed %(refname)s line: '%(line)s'") % - ({'refname': refname, 'line':self._currentline}) - ) # pragma: no cover - ref = matches.group(1) - self._advance_currentline() - return ref - - def _parse_user(self, usertype): - """ - Get user name, email, datestamp from current-line. Current-line will - be advanced. - """ - user_regex = self._user_regexes[usertype] - (name, email, when) = user_regex.match(self._currentline).groups() - - self._advance_currentline() - return (name, email, when) - - def _parse_data(self): - """ - Reads data from _input. Current-line will be advanced until it is beyond - the data. - """ - fields = self._currentline.split() - assert fields[0] == b'data' - size = int(fields[1]) - data = self._input.read(size) - self._advance_currentline() - if self._currentline == b'\n': - self._advance_currentline() - return data - - def _parse_blob(self): - """ - Parse input data into a Blob object. Once the Blob has been created, it - will be handed off to the appropriate callbacks. Current-line will be - advanced until it is beyond this blob's data. The Blob will be dumped - to _output once everything else is done (unless it has been skipped by - the callback). - """ - # Parse the Blob - self._advance_currentline() - id_ = self._parse_optional_mark() - - original_id = None - if self._currentline.startswith(b'original-oid'): - original_id = self._parse_original_id(); - - data = self._parse_data() - if self._currentline == b'\n': - self._advance_currentline() - - # Create the blob - blob = Blob(data, original_id) - - # If fast-export text had a mark for this blob, need to make sure this - # mark translates to the blob's true id. - if id_: - blob.old_id = id_ - _IDS.record_rename(id_, blob.id) - - # Call any user callback to allow them to use/modify the blob - if self._blob_callback: - self._blob_callback(blob) - - # Now print the resulting blob - if not blob.dumped: - blob.dump(self._output) - - def _parse_reset(self): - """ - Parse input data into a Reset object. Once the Reset has been created, - it will be handed off to the appropriate callbacks. Current-line will - be advanced until it is beyond the reset data. The Reset will be dumped - to _output once everything else is done (unless it has been skipped by - the callback). - """ - # Parse the Reset - ref = self._parse_ref_line(b'reset') - self._exported_refs.add(ref) - ignoreme, from_ref = self._parse_optional_parent_ref(b'from') - if self._currentline == b'\n': - self._advance_currentline() - - # fast-export likes to print extraneous resets that serve no purpose. - # While we could continue processing such resets, that is a waste of - # resources. Also, we want to avoid recording that this ref was - # seen in such cases, since this ref could be rewritten to nothing. - if not from_ref: - self._latest_commit.pop(ref, None) - self._latest_orig_commit.pop(ref, None) - return - - # Create the reset - reset = Reset(ref, from_ref) - - # Call any user callback to allow them to modify the reset - if self._reset_callback: - self._reset_callback(reset) - - # Update metadata - self._latest_commit[reset.ref] = reset.from_ref - self._latest_orig_commit[reset.ref] = reset.from_ref - - # Now print the resulting reset - if not reset.dumped: - self._imported_refs.add(reset.ref) - reset.dump(self._output) - - def _parse_commit(self): - """ - Parse input data into a Commit object. Once the Commit has been created, - it will be handed off to the appropriate callbacks. Current-line will - be advanced until it is beyond the commit data. The Commit will be dumped - to _output once everything else is done (unless it has been skipped by - the callback OR the callback has removed all file-changes from the commit). - """ - # Parse the Commit. This may look involved, but it's pretty simple; it only - # looks bad because a commit object contains many pieces of data. - branch = self._parse_ref_line(b'commit') - self._exported_refs.add(branch) - id_ = self._parse_optional_mark() - - original_id = None - if self._currentline.startswith(b'original-oid'): - original_id = self._parse_original_id(); - - author_name = None - author_email = None - if self._currentline.startswith(b'author'): - (author_name, author_email, author_date) = self._parse_user(b'author') - - (committer_name, committer_email, committer_date) = \ - self._parse_user(b'committer') - - if not author_name and not author_email: - (author_name, author_email, author_date) = \ - (committer_name, committer_email, committer_date) - - encoding = None - if self._currentline.startswith(b'encoding '): - encoding = self._parse_encoding() - - commit_msg = self._parse_data() - - pinfo = [self._parse_optional_parent_ref(b'from')] - # Due to empty pruning, we can have real 'from' and 'merge' lines that - # due to commit rewriting map to a parent of None. We need to record - # 'from' if its non-None, and we need to parse all 'merge' lines. - while self._currentline.startswith(b'merge '): - pinfo.append(self._parse_optional_parent_ref(b'merge')) - orig_parents, parents = [list(tmp) for tmp in zip(*pinfo)] - - # No parents is oddly represented as [None] instead of [], due to the - # special 'from' handling. Convert it here to a more canonical form. - if parents == [None]: - parents = [] - if orig_parents == [None]: - orig_parents = [] - - # fast-import format is kinda stupid in that it allows implicit parents - # based on the branch name instead of requiring them to be specified by - # 'from' directives. The only way to get no parent is by using a reset - # directive first, which clears the latest_commit_for_this_branch tracking. - if not orig_parents and self._latest_commit.get(branch): - parents = [self._latest_commit[branch]] - if not orig_parents and self._latest_orig_commit.get(branch): - orig_parents = [self._latest_orig_commit[branch]] - - # Get the list of file changes - file_changes = [] - file_change = self._parse_optional_filechange() - had_file_changes = file_change is not None - while file_change: - if not (type(file_change) == bytes and file_change == b'skipped'): - file_changes.append(file_change) - file_change = self._parse_optional_filechange() - if self._currentline == b'\n': - self._advance_currentline() - - # Okay, now we can finally create the Commit object - commit = Commit(branch, - author_name, author_email, author_date, - committer_name, committer_email, committer_date, - commit_msg, file_changes, parents, original_id, encoding) - - # If fast-export text had a mark for this commit, need to make sure this - # mark translates to the commit's true id. - if id_: - commit.old_id = id_ - _IDS.record_rename(id_, commit.id) - - # Call any user callback to allow them to modify the commit - aux_info = {'orig_parents': orig_parents, - 'had_file_changes': had_file_changes} - if self._commit_callback: - self._commit_callback(commit, aux_info) - - # Now print the resulting commit, or if prunable skip it - self._latest_orig_commit[branch] = commit.id - if not (commit.old_id or commit.id) in _SKIPPED_COMMITS: - self._latest_commit[branch] = commit.id - if not commit.dumped: - self._imported_refs.add(commit.branch) - commit.dump(self._output) - - def _parse_tag(self): - """ - Parse input data into a Tag object. Once the Tag has been created, - it will be handed off to the appropriate callbacks. Current-line will - be advanced until it is beyond the tag data. The Tag will be dumped - to _output once everything else is done (unless it has been skipped by - the callback). - """ - # Parse the Tag - tag = self._parse_ref_line(b'tag') - self._exported_refs.add(b'refs/tags/'+tag) - id_ = self._parse_optional_mark() - ignoreme, from_ref = self._parse_optional_parent_ref(b'from') - - original_id = None - if self._currentline.startswith(b'original-oid'): - original_id = self._parse_original_id(); - - tagger_name, tagger_email, tagger_date = None, None, None - if self._currentline.startswith(b'tagger'): - (tagger_name, tagger_email, tagger_date) = self._parse_user(b'tagger') - tag_msg = self._parse_data() - if self._currentline == b'\n': - self._advance_currentline() - - # Create the tag - tag = Tag(tag, from_ref, - tagger_name, tagger_email, tagger_date, tag_msg, - original_id) - - # If fast-export text had a mark for this tag, need to make sure this - # mark translates to the tag's true id. - if id_: - tag.old_id = id_ - _IDS.record_rename(id_, tag.id) - - # Call any user callback to allow them to modify the tag - if self._tag_callback: - self._tag_callback(tag) - - # The tag might not point at anything that still exists (self.from_ref - # will be None if the commit it pointed to and all its ancestors were - # pruned due to being empty) - if tag.from_ref: - # Print out this tag's information - if not tag.dumped: - self._imported_refs.add(b'refs/tags/'+tag.ref) - tag.dump(self._output) - else: - tag.skip() - - def _parse_progress(self): - """ - Parse input data into a Progress object. Once the Progress has - been created, it will be handed off to the appropriate - callbacks. Current-line will be advanced until it is beyond the - progress data. The Progress will be dumped to _output once - everything else is done (unless it has been skipped by the callback). - """ - # Parse the Progress - message = self._parse_ref_line(b'progress') - if self._currentline == b'\n': - self._advance_currentline() - - # Create the progress message - progress = Progress(message) - - # Call any user callback to allow them to modify the progress messsage - if self._progress_callback: - self._progress_callback(progress) - - # NOTE: By default, we do NOT print the progress message; git - # fast-import would write it to fast_import_pipes which could mess with - # our parsing of output from the 'ls' and 'get-mark' directives we send - # to fast-import. If users want these messages, they need to process - # and handle them in the appropriate callback above. - - def _parse_checkpoint(self): - """ - Parse input data into a Checkpoint object. Once the Checkpoint has - been created, it will be handed off to the appropriate - callbacks. Current-line will be advanced until it is beyond the - checkpoint data. The Checkpoint will be dumped to _output once - everything else is done (unless it has been skipped by the callback). - """ - # Parse the Checkpoint - self._advance_currentline() - if self._currentline == b'\n': - self._advance_currentline() - - # Create the checkpoint - checkpoint = Checkpoint() - - # Call any user callback to allow them to drop the checkpoint - if self._checkpoint_callback: - self._checkpoint_callback(checkpoint) - - # NOTE: By default, we do NOT print the checkpoint message; although it - # we would only realistically get them with --stdin, the fact that we - # are filtering makes me think the checkpointing is less likely to be - # reasonable. In fact, I don't think it's necessary in general. If - # users do want it, they should process it in the checkpoint_callback. - - def _parse_literal_command(self): - """ - Parse literal command. Then just dump the line as is. - """ - # Create the literal command object - command = LiteralCommand(self._currentline) - self._advance_currentline() - - # Now print the resulting literal command - if not command.dumped: - command.dump(self._output) - - def insert(self, obj): - assert not obj.dumped - obj.dump(self._output) - if type(obj) == Commit: - self._imported_refs.add(obj.branch) - elif type(obj) in (Reset, Tag): - self._imported_refs.add(obj.ref) - - def run(self, input, output): - """ - This method filters fast export output. - """ - # Set input. If no args provided, use stdin. - self._input = input - self._output = output - - # Run over the input and do the filtering - self._advance_currentline() - while self._currentline: - if self._currentline.startswith(b'blob'): - self._parse_blob() - elif self._currentline.startswith(b'reset'): - self._parse_reset() - elif self._currentline.startswith(b'commit'): - self._parse_commit() - elif self._currentline.startswith(b'tag'): - self._parse_tag() - elif self._currentline.startswith(b'progress'): - self._parse_progress() - elif self._currentline.startswith(b'checkpoint'): - self._parse_checkpoint() - elif self._currentline.startswith(b'feature'): - self._parse_literal_command() - elif self._currentline.startswith(b'option'): - self._parse_literal_command() - elif self._currentline.startswith(b'done'): - if self._done_callback: - self._done_callback() - self._parse_literal_command() - # Prevent confusion from others writing additional stuff that'll just - # be ignored - self._output.close() - elif self._currentline.startswith(b'#'): - self._parse_literal_command() - elif self._currentline.startswith(b'get-mark') or \ - self._currentline.startswith(b'cat-blob') or \ - self._currentline.startswith(b'ls'): - raise SystemExit(_("Unsupported command: '%s'") % self._currentline) - else: - raise SystemExit(_("Could not parse line: '%s'") % self._currentline) - - def get_exported_and_imported_refs(self): - return self._exported_refs, self._imported_refs - -def record_id_rename(old_id, new_id): - """ - Register a new translation - """ - handle_transitivity = True - _IDS.record_rename(old_id, new_id, handle_transitivity) - -# Internal globals -_IDS = _IDs() -_SKIPPED_COMMITS = set() -HASH_TO_ID = {} -ID_TO_HASH = {} - -class SubprocessWrapper(object): - @staticmethod - def decodify(args): - if type(args) == str: - return args - else: - assert type(args) == list - return [decode(x) if type(x)==bytes else x for x in args] - - @staticmethod - def call(*args, **kwargs): - if 'cwd' in kwargs: - kwargs['cwd'] = decode(kwargs['cwd']) - return subprocess.call(SubprocessWrapper.decodify(*args), **kwargs) - - @staticmethod - def check_output(*args, **kwargs): - if 'cwd' in kwargs: - kwargs['cwd'] = decode(kwargs['cwd']) - return subprocess.check_output(SubprocessWrapper.decodify(*args), **kwargs) - - @staticmethod - def check_call(*args, **kwargs): # pragma: no cover # used by filter-lamely - if 'cwd' in kwargs: - kwargs['cwd'] = decode(kwargs['cwd']) - return subprocess.check_call(SubprocessWrapper.decodify(*args), **kwargs) - - @staticmethod - def Popen(*args, **kwargs): - if 'cwd' in kwargs: - kwargs['cwd'] = decode(kwargs['cwd']) - return subprocess.Popen(SubprocessWrapper.decodify(*args), **kwargs) - -subproc = subprocess -if platform.system() == 'Windows' or 'PRETEND_UNICODE_ARGS' in os.environ: - subproc = SubprocessWrapper - -class GitUtils(object): - @staticmethod - def get_commit_count(repo, *args): - """ - Return the number of commits that have been made on repo. - """ - if not args: - args = ['--all'] - if len(args) == 1 and isinstance(args[0], list): - args = args[0] - p = subproc.Popen(["git", "rev-list", "--count"] + args, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd=repo) - if p.wait() != 0: - raise SystemExit(_("%s does not appear to be a valid git repository") - % decode(repo)) - return int(p.stdout.read()) - - @staticmethod - def get_total_objects(repo): - """ - Return the number of objects (both packed and unpacked) - """ - p1 = subproc.Popen(["git", "count-objects", "-v"], - stdout=subprocess.PIPE, cwd=repo) - lines = p1.stdout.read().splitlines() - # Return unpacked objects + packed-objects - return int(lines[0].split()[1]) + int(lines[2].split()[1]) - - @staticmethod - def is_repository_bare(repo_working_dir): - out = subproc.check_output('git rev-parse --is-bare-repository'.split(), - cwd=repo_working_dir) - return (out.strip() == b'true') - - @staticmethod - def determine_git_dir(repo_working_dir): - d = subproc.check_output('git rev-parse --git-dir'.split(), - cwd=repo_working_dir).strip() - if repo_working_dir==b'.' or d.startswith(b'/'): - return d - return os.path.join(repo_working_dir, d) - - @staticmethod - def get_refs(repo_working_dir): - try: - output = subproc.check_output('git show-ref'.split(), - cwd=repo_working_dir) - except subprocess.CalledProcessError as e: - # If error code is 1, there just aren't any refs; i.e. new repo. - # If error code is other than 1, some other error (e.g. not a git repo) - if e.returncode != 1: - raise SystemExit('fatal: {}'.format(e)) - output = '' - return dict(reversed(x.split()) for x in output.splitlines()) - - @staticmethod - def get_blob_sizes(quiet = False): - blob_size_progress = ProgressWriter() - num_blobs = 0 - processed_blobs_msg = _("Processed %d blob sizes") - - # Get sizes of blobs by sha1 - cmd = '--batch-check=%(objectname) %(objecttype) ' + \ - '%(objectsize) %(objectsize:disk)' - cf = subproc.Popen(['git', 'cat-file', '--batch-all-objects', cmd], - bufsize = -1, - stdout = subprocess.PIPE) - unpacked_size = {} - packed_size = {} - for line in cf.stdout: - sha, objtype, objsize, objdisksize = line.split() - objsize, objdisksize = int(objsize), int(objdisksize) - if objtype == b'blob': - unpacked_size[sha] = objsize - packed_size[sha] = objdisksize - num_blobs += 1 - if not quiet: - blob_size_progress.show(processed_blobs_msg % num_blobs) - cf.wait() - if not quiet: - blob_size_progress.finish() - return unpacked_size, packed_size - - @staticmethod - def get_file_changes(repo, parent_hash, commit_hash): - """ - Return a FileChanges list with the differences between parent_hash - and commit_hash - """ - file_changes = [] - - cmd = ["git", "diff-tree", "-r", parent_hash, commit_hash] - output = subproc.check_output(cmd, cwd=repo) - for line in output.splitlines(): - fileinfo, path = line.split(b'\t', 1) - if path.startswith(b'"'): - path = PathQuoting.dequote(path) - oldmode, mode, oldhash, newhash, changetype = fileinfo.split() - if changetype == b'D': - file_changes.append(FileChange(b'D', path)) - elif changetype in (b'A', b'M', b'T'): - identifier = HASH_TO_ID.get(newhash, newhash) - file_changes.append(FileChange(b'M', path, identifier, mode)) - else: # pragma: no cover - raise SystemExit("Unknown change type for line {}".format(line)) - - return file_changes - - @staticmethod - def print_my_version(): - with open(__file__, 'br') as f: - contents = f.read() - # If people replaced @@LOCALEDIR@@ string to point at their local - # directory, undo it so we can get original source version. - contents = re.sub(br'\A#\!.*', - br'#!/usr/bin/env python3', contents) - contents = re.sub(br'(\("GIT_TEXTDOMAINDIR"\) or ").*"', - br'\1@@LOCALEDIR@@"', contents) - - cmd = 'git hash-object --stdin'.split() - version = subproc.check_output(cmd, input=contents).strip() - print(decode(version[0:12])) - -class FilteringOptions(object): - default_replace_text = b'***REMOVED***' - class AppendFilter(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): - user_path = values - suffix = option_string[len('--path-'):] or 'match' - if suffix.startswith('rename'): - mod_type = 'rename' - match_type = option_string[len('--path-rename-'):] or 'match' - values = values.split(b':') - if len(values) != 2: - raise SystemExit(_("Error: --path-rename expects one colon in its" - " argument: .")) - if values[0] and values[1] and not ( - values[0].endswith(b'/') == values[1].endswith(b'/')): - raise SystemExit(_("Error: With --path-rename, if OLD_NAME and " - "NEW_NAME are both non-empty and either ends " - "with a slash then both must.")) - if any(v.startswith(b'/') for v in values): - raise SystemExit(_("Error: Pathnames cannot begin with a '/'")) - components = values[0].split(b'/') + values[1].split(b'/') - else: - mod_type = 'filter' - match_type = suffix - components = values.split(b'/') - if values.startswith(b'/'): - raise SystemExit(_("Error: Pathnames cannot begin with a '/'")) - for illegal_path in [b'.', b'..']: - if illegal_path in components: - raise SystemExit(_("Error: Invalid path component '%s' found in '%s'") - % (decode(illegal_path), decode(user_path))) - if match_type == 'regex': - values = re.compile(values) - items = getattr(namespace, self.dest, []) or [] - items.append((mod_type, match_type, values)) - if (match_type, mod_type) == ('glob', 'filter'): - if not values.endswith(b'*'): - extension = b'*' if values.endswith(b'/') else b'/*' - items.append((mod_type, match_type, values+extension)) - setattr(namespace, self.dest, items) - - class HelperFilter(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): - af = FilteringOptions.AppendFilter(dest='path_changes', - option_strings=None) - dirname = values if values[-1:] == b'/' else values+b'/' - if option_string == '--subdirectory-filter': - af(parser, namespace, dirname, '--path-match') - af(parser, namespace, dirname+b':', '--path-rename') - elif option_string == '--to-subdirectory-filter': - af(parser, namespace, b':'+dirname, '--path-rename') - else: - raise SystemExit(_("Error: HelperFilter given invalid option_string: %s") - % option_string) # pragma: no cover - - class FileWithPathsFilter(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): - if not namespace.path_changes: - namespace.path_changes = [] - namespace.path_changes += FilteringOptions.get_paths_from_file(values) - - @staticmethod - def create_arg_parser(): - # Include usage in the summary, so we can put the description first - summary = _('''Rewrite (or analyze) repository history - - git-filter-repo destructively rewrites history (unless --analyze or - --dry-run are given) according to specified rules. It refuses to do any - rewriting unless either run from a clean fresh clone, or --force was - given. - - Basic Usage: - git-filter-repo --analyze - git-filter-repo [FILTER/RENAME/CONTROL OPTIONS] - - See EXAMPLES section for details. - ''').rstrip() - - # Provide a long helpful examples section - example_text = _('''CALLBACKS - - All callback functions are of the same general format. For a command line - argument like - --foo-callback 'BODY' - - the following code will be compiled and called: - def foo_callback(foo): - BODY - - Thus, to replace 'Jon' with 'John' in author/committer/tagger names: - git filter-repo --name-callback 'return name.replace(b"Jon", b"John")' - - To remove all 'Tested-by' tags in commit (or tag) messages: - git filter-repo --message-callback 'return re.sub(br"\\nTested-by:.*", "", message)' - - To remove all .DS_Store files: - git filter-repo --filename-callback 'return None if os.path.basename(filename) == b".DS_Store" else filename' - - Note that if BODY resolves to a filename, then the contents of that file - will be used as the BODY in the callback function. - - For more detailed examples and explanations AND caveats, see - https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html#CALLBACKS - -EXAMPLES - - To get a bunch of reports mentioning renames that have occurred in - your repo and listing sizes of objects aggregated by any of path, - directory, extension, or blob-id: - git filter-repo --analyze - - (These reports can help you choose how to filter your repo; it can - be useful to re-run this command after filtering to regenerate the - report and verify the changes look correct.) - - To extract the history that touched just 'guides' and 'tools/releases': - git filter-repo --path guides/ --path tools/releases - - To remove foo.zip and bar/baz/zips from every revision in history: - git filter-repo --path foo.zip --path bar/baz/zips/ --invert-paths - - To replace the text 'password' with 'p455w0rd': - git filter-repo --replace-text <(echo "password==>p455w0rd") - - To use the current version of the .mailmap file to update authors, - committers, and taggers throughout history and make it permanent: - git filter-repo --use-mailmap - - To extract the history of 'src/', rename all files to have a new leading - directory 'my-module' (e.g. src/foo.java -> my-module/src/foo.java), and - add a 'my-module-' prefix to all tags: - git filter-repo --path src/ --to-subdirectory-filter my-module --tag-rename '':'my-module-' - - For more detailed examples and explanations, see - https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html#EXAMPLES''') - - # Create the basic parser - parser = argparse.ArgumentParser(description=summary, - usage = argparse.SUPPRESS, - add_help = False, - epilog = example_text, - formatter_class=argparse.RawDescriptionHelpFormatter) - - analyze = parser.add_argument_group(title=_("Analysis")) - analyze.add_argument('--analyze', action='store_true', - help=_("Analyze repository history and create a report that may be " - "useful in determining what to filter in a subsequent run. " - "Will not modify your repo.")) - analyze.add_argument('--report-dir', - metavar='DIR_OR_FILE', - type=os.fsencode, - dest='report_dir', - help=_("Directory to write report, defaults to GIT_DIR/filter_repo/analysis," - "refuses to run if exists, --force delete existing dir first.")) - - path = parser.add_argument_group(title=_("Filtering based on paths " - "(see also --filename-callback)"), - description=textwrap.dedent(_(""" - These options specify the paths to select. Note that much like git - itself, renames are NOT followed so you may need to specify multiple - paths, e.g. `--path olddir/ --path newdir/` - """[1:]))) - - path.add_argument('--invert-paths', action='store_false', dest='inclusive', - help=_("Invert the selection of files from the specified " - "--path-{match,glob,regex} options below, i.e. only select " - "files matching none of those options.")) - - path.add_argument('--path-match', '--path', metavar='DIR_OR_FILE', - type=os.fsencode, - action=FilteringOptions.AppendFilter, dest='path_changes', - help=_("Exact paths (files or directories) to include in filtered " - "history. Multiple --path options can be specified to get " - "a union of paths.")) - path.add_argument('--path-glob', metavar='GLOB', type=os.fsencode, - action=FilteringOptions.AppendFilter, dest='path_changes', - help=_("Glob of paths to include in filtered history. Multiple " - "--path-glob options can be specified to get a union of " - "paths.")) - path.add_argument('--path-regex', metavar='REGEX', type=os.fsencode, - action=FilteringOptions.AppendFilter, dest='path_changes', - help=_("Regex of paths to include in filtered history. Multiple " - "--path-regex options can be specified to get a union of " - "paths")) - path.add_argument('--use-base-name', action='store_true', - help=_("Match on file base name instead of full path from the top " - "of the repo. Incompatible with --path-rename, and " - "incompatible with matching against directory names.")) - - rename = parser.add_argument_group(title=_("Renaming based on paths " - "(see also --filename-callback)")) - rename.add_argument('--path-rename', '--path-rename-match', - metavar='OLD_NAME:NEW_NAME', dest='path_changes', type=os.fsencode, - action=FilteringOptions.AppendFilter, - help=_("Path to rename; if filename or directory matches OLD_NAME " - "rename to NEW_NAME. Multiple --path-rename options can be " - "specified. NOTE: If you combine filtering options with " - "renaming ones, do not rely on a rename argument to select " - "paths; you also need a filter to select them.")) - - helpers = parser.add_argument_group(title=_("Path shortcuts")) - helpers.add_argument('--paths-from-file', metavar='FILENAME', - type=os.fsencode, - action=FilteringOptions.FileWithPathsFilter, dest='path_changes', - help=_("Specify several path filtering and renaming directives, one " - "per line. Lines with '==>' in them specify path renames, " - "and lines can begin with 'literal:' (the default), 'glob:', " - "or 'regex:' to specify different matching styles. Blank " - "lines and lines starting with a '#' are ignored.")) - helpers.add_argument('--subdirectory-filter', metavar='DIRECTORY', - action=FilteringOptions.HelperFilter, type=os.fsencode, - help=_("Only look at history that touches the given subdirectory " - "and treat that directory as the project root. Equivalent " - "to using '--path DIRECTORY/ --path-rename DIRECTORY/:'")) - helpers.add_argument('--to-subdirectory-filter', metavar='DIRECTORY', - action=FilteringOptions.HelperFilter, type=os.fsencode, - help=_("Treat the project root as instead being under DIRECTORY. " - "Equivalent to using '--path-rename :DIRECTORY/'")) - - contents = parser.add_argument_group(title=_("Content editing filters " - "(see also --blob-callback)")) - contents.add_argument('--replace-text', metavar='EXPRESSIONS_FILE', - help=_("A file with expressions that, if found, will be replaced. " - "By default, each expression is treated as literal text, " - "but 'regex:' and 'glob:' prefixes are supported. You can " - "end the line with '==>' and some replacement text to " - "choose a replacement choice other than the default of '{}'." - .format(decode(FilteringOptions.default_replace_text)))) - contents.add_argument('--strip-blobs-bigger-than', metavar='SIZE', - dest='max_blob_size', default=0, - help=_("Strip blobs (files) bigger than specified size (e.g. '5M', " - "'2G', etc)")) - contents.add_argument('--strip-blobs-with-ids', metavar='BLOB-ID-FILENAME', - help=_("Read git object ids from each line of the given file, and " - "strip all of them from history")) - - refrename = parser.add_argument_group(title=_("Renaming of refs " - "(see also --refname-callback)")) - refrename.add_argument('--tag-rename', metavar='OLD:NEW', type=os.fsencode, - help=_("Rename tags starting with OLD to start with NEW. For " - "example, --tag-rename foo:bar will rename tag foo-1.2.3 " - "to bar-1.2.3; either OLD or NEW can be empty.")) - - messages = parser.add_argument_group(title=_("Filtering of commit messages " - "(see also --message-callback)")) - messages.add_argument('--replace-message', metavar='EXPRESSIONS_FILE', - help=_("A file with expressions that, if found in commit messages, " - "will be replaced. This file uses the same syntax as " - "--replace-text.")) - messages.add_argument('--preserve-commit-hashes', action='store_true', - help=_("By default, since commits are rewritten and thus gain new " - "hashes, references to old commit hashes in commit messages " - "are replaced with new commit hashes (abbreviated to the same " - "length as the old reference). Use this flag to turn off " - "updating commit hashes in commit messages.")) - messages.add_argument('--preserve-commit-encoding', action='store_true', - help=_("Do not reencode commit messages into UTF-8. By default, if " - "the commit object specifies an encoding for the commit " - "message, the message is re-encoded into UTF-8.")) - - people = parser.add_argument_group(title=_("Filtering of names & emails " - "(see also --name-callback " - "and --email-callback)")) - people.add_argument('--mailmap', dest='mailmap', metavar='FILENAME', - type=os.fsencode, - help=_("Use specified mailmap file (see git-shortlog(1) for " - "details on the format) when rewriting author, committer, " - "and tagger names and emails. If the specified file is " - "part of git history, historical versions of the file will " - "be ignored; only the current contents are consulted.")) - people.add_argument('--use-mailmap', dest='mailmap', - action='store_const', const=b'.mailmap', - help=_("Same as: '--mailmap .mailmap' ")) - - parents = parser.add_argument_group(title=_("Parent rewriting")) - parents.add_argument('--replace-refs', default=None, - choices=['delete-no-add', 'delete-and-add', - 'update-no-add', 'update-or-add', - 'update-and-add'], - help=_("Replace refs (see git-replace(1)) are used to rewrite " - "parents (unless turned off by the usual git mechanism); this " - "flag specifies what do do with those refs afterward. " - "Replace refs can either be deleted or updated to point at new " - "commit hashes. Also, new replace refs can be added for each " - "commit rewrite. With 'update-or-add', new replace refs are " - "only added for commit rewrites that aren't used to update an " - "existing replace ref. default is 'update-and-add' if " - "$GIT_DIR/filter-repo/already_ran does not exist; " - "'update-or-add' otherwise.")) - parents.add_argument('--prune-empty', default='auto', - choices=['always', 'auto', 'never'], - help=_("Whether to prune empty commits. 'auto' (the default) means " - "only prune commits which become empty (not commits which were " - "empty in the original repo, unless their parent was pruned). " - "When the parent of a commit is pruned, the first non-pruned " - "ancestor becomes the new parent.")) - parents.add_argument('--prune-degenerate', default='auto', - choices=['always', 'auto', 'never'], - help=_("Since merge commits are needed for history topology, they " - "are typically exempt from pruning. However, they can become " - "degenerate with the pruning of other commits (having fewer " - "than two parents, having one commit serve as both parents, or " - "having one parent as the ancestor of the other.) If such " - "merge commits have no file changes, they can be pruned. The " - "default ('auto') is to only prune empty merge commits which " - "become degenerate (not which started as such).")) - parents.add_argument('--no-ff', action='store_true', - help=_("Even if the first parent is or becomes an ancestor of another " - "parent, do not prune it. This modifies how " - "--prune-degenerate behaves, and may be useful in projects who " - "always use merge --no-ff.")) - - callback = parser.add_argument_group(title=_("Generic callback code snippets")) - callback.add_argument('--filename-callback', metavar="FUNCTION_BODY_OR_FILE", - help=_("Python code body for processing filenames; see CALLBACKS " - "sections below.")) - callback.add_argument('--message-callback', metavar="FUNCTION_BODY_OR_FILE", - help=_("Python code body for processing messages (both commit " - "messages and tag messages); see CALLBACKS section below.")) - callback.add_argument('--name-callback', metavar="FUNCTION_BODY_OR_FILE", - help=_("Python code body for processing names of people; see " - "CALLBACKS section below.")) - callback.add_argument('--email-callback', metavar="FUNCTION_BODY_OR_FILE", - help=_("Python code body for processing emails addresses; see " - "CALLBACKS section below.")) - callback.add_argument('--refname-callback', metavar="FUNCTION_BODY_OR_FILE", - help=_("Python code body for processing refnames; see CALLBACKS " - "section below.")) - - callback.add_argument('--blob-callback', metavar="FUNCTION_BODY_OR_FILE", - help=_("Python code body for processing blob objects; see " - "CALLBACKS section below.")) - callback.add_argument('--commit-callback', metavar="FUNCTION_BODY_OR_FILE", - help=_("Python code body for processing commit objects; see " - "CALLBACKS section below.")) - callback.add_argument('--tag-callback', metavar="FUNCTION_BODY_OR_FILE", - help=_("Python code body for processing tag objects; see CALLBACKS " - "section below.")) - callback.add_argument('--reset-callback', metavar="FUNCTION_BODY_OR_FILE", - help=_("Python code body for processing reset objects; see " - "CALLBACKS section below.")) - - desc = _( - "Specifying alternate source or target locations implies --partial,\n" - "except that the normal default for --replace-refs is used. However,\n" - "unlike normal uses of --partial, this doesn't risk mixing old and new\n" - "history since the old and new histories are in different repositories.") - location = parser.add_argument_group(title=_("Location to filter from/to"), - description=desc) - location.add_argument('--source', type=os.fsencode, - help=_("Git repository to read from")) - location.add_argument('--target', type=os.fsencode, - help=_("Git repository to overwrite with filtered history")) - - misc = parser.add_argument_group(title=_("Miscellaneous options")) - misc.add_argument('--help', '-h', action='store_true', - help=_("Show this help message and exit.")) - misc.add_argument('--version', action='store_true', - help=_("Display filter-repo's version and exit.")) - misc.add_argument('--force', '-f', action='store_true', - help=_("Rewrite repository history even if the current repo does not " - "look like a fresh clone. History rewriting is irreversible " - "(and includes immediate pruning of reflogs and old objects), " - "so be cautious about using this flag.")) - misc.add_argument('--partial', action='store_true', - help=_("Do a partial history rewrite, resulting in the mixture of " - "old and new history. This implies a default of " - "update-no-add for --replace-refs, disables rewriting " - "refs/remotes/origin/* to refs/heads/*, disables removing " - "of the 'origin' remote, disables removing unexported refs, " - "disables expiring the reflog, and disables the automatic " - "post-filter gc. Also, this modifies --tag-rename and " - "--refname-callback options such that instead of replacing " - "old refs with new refnames, it will instead create new " - "refs and keep the old ones around. Use with caution.")) - # WARNING: --refs presents a problem with become-degenerate pruning: - # * Excluding a commit also excludes its ancestors so when some other - # commit has an excluded ancestor as a parent we have no way of - # knowing what it is an ancestor of without doing a special - # full-graph walk. - misc.add_argument('--refs', nargs='+', - help=_("Limit history rewriting to the specified refs. Implies " - "--partial. In addition to the normal caveats of --partial " - "(mixing old and new history, no automatic remapping of " - "refs/remotes/origin/* to refs/heads/*, etc.), this also may " - "cause problems for pruning of degenerate empty merge " - "commits when negative revisions are specified.")) - - misc.add_argument('--dry-run', action='store_true', - help=_("Do not change the repository. Run `git fast-export` and " - "filter its output, and save both the original and the " - "filtered version for comparison. This also disables " - "rewriting commit messages due to not knowing new commit " - "IDs and disables filtering of some empty commits due to " - "inability to query the fast-import backend." )) - misc.add_argument('--debug', action='store_true', - help=_("Print additional information about operations being " - "performed and commands being run. When used together " - "with --dry-run, also show extra information about what " - "would be run.")) - # WARNING: --state-branch has some problems: - # * It does not work well with manually inserted objects (user creating - # Blob() or Commit() or Tag() objects and calling - # RepoFilter.insert(obj) on them). - # * It does not work well with multiple source or multiple target repos - # * It doesn't work so well with pruning become-empty commits (though - # --refs doesn't work so well with it either) - # These are probably fixable, given some work (e.g. re-importing the - # graph at the beginning to get the AncestryGraph right, doing our own - # export of marks instead of using fast-export --export-marks, etc.), but - # for now just hide the option. - misc.add_argument('--state-branch', - #help=_("Enable incremental filtering by saving the mapping of old " - # "to new objects to the specified branch upon exit, and" - # "loading that mapping from that branch (if it exists) " - # "upon startup.")) - help=argparse.SUPPRESS) - misc.add_argument('--stdin', action='store_true', - help=_("Instead of running `git fast-export` and filtering its " - "output, filter the fast-export stream from stdin. The " - "stdin must be in the expected input format (e.g. it needs " - "to include original-oid directives).")) - misc.add_argument('--quiet', action='store_true', - help=_("Pass --quiet to other git commands called")) - return parser - - @staticmethod - def sanity_check_args(args): - if args.analyze and args.path_changes: - raise SystemExit(_("Error: --analyze is incompatible with --path* flags; " - "it's a read-only operation.")) - if args.analyze and args.stdin: - raise SystemExit(_("Error: --analyze is incompatible with --stdin.")) - # If no path_changes are found, initialize with empty list but mark as - # not inclusive so that all files match - if args.path_changes == None: - args.path_changes = [] - args.inclusive = False - else: - # Similarly, if we have no filtering paths, then no path should be - # filtered out. Based on how newname() works, the easiest way to - # achieve that is setting args.inclusive to False. - if not any(x[0] == 'filter' for x in args.path_changes): - args.inclusive = False - # Also check for incompatible --use-base-name and --path-rename flags. - if args.use_base_name: - if any(x[0] == 'rename' for x in args.path_changes): - raise SystemExit(_("Error: --use-base-name and --path-rename are " - "incompatible.")) - # Also throw some sanity checks on git version here; - # PERF: remove these checks once new enough git versions are common - p = subproc.Popen('git fast-export -h'.split(), - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = p.stdout.read() - if b'--anonymize-map' not in output: # pragma: no cover - global date_format_permissive - date_format_permissive = False - if b'--mark-tags' not in output: # pragma: no cover - global write_marks - write_marks = False - if args.state_branch: - # We need a version of git-fast-export with --mark-tags - raise SystemExit(_("Error: need git >= 2.24.0")) - if b'--reencode' not in output: # pragma: no cover - if args.preserve_commit_encoding: - # We need a version of git-fast-export with --reencode - raise SystemExit(_("Error: need git >= 2.23.0")) - else: - # Set args.preserve_commit_encoding to None which we'll check for later - # to avoid passing --reencode=yes to fast-export (that option was the - # default prior to git-2.23) - args.preserve_commit_encoding = None - # If we don't have fast-exoprt --reencode, we may also be missing - # diff-tree --combined-all-paths, which is even more important... - p = subproc.Popen('git diff-tree -h'.split(), - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = p.stdout.read() - if b'--combined-all-paths' not in output: - # We need a version of git-diff-tree with --combined-all-paths - raise SystemExit(_("Error: need git >= 2.22.0")) - # End of sanity checks on git version - if args.max_blob_size: - suffix = args.max_blob_size[-1] - if suffix not in '1234567890': - mult = {'K': 1024, 'M': 1024**2, 'G': 1024**3} - if suffix not in mult: - raise SystemExit(_("Error: could not parse --strip-blobs-bigger-than" - " argument %s") - % args.max_blob_size) - args.max_blob_size = int(args.max_blob_size[0:-1]) * mult[suffix] - else: - args.max_blob_size = int(args.max_blob_size) - - @staticmethod - def get_replace_text(filename): - replace_literals = [] - replace_regexes = [] - with open(filename, 'br') as f: - for line in f: - line = line.rstrip(b'\r\n') - - # Determine the replacement - replacement = FilteringOptions.default_replace_text - if b'==>' in line: - line, replacement = line.rsplit(b'==>', 1) - - # See if we need to match via regex - regex = None - if line.startswith(b'regex:'): - regex = line[6:] - elif line.startswith(b'glob:'): - regex = glob_to_regex(line[5:]) - if regex: - replace_regexes.append((re.compile(regex), replacement)) - else: - # Otherwise, find the literal we need to replace - if line.startswith(b'literal:'): - line = line[8:] - if not line: - continue - replace_literals.append((line, replacement)) - return {'literals': replace_literals, 'regexes': replace_regexes} - - @staticmethod - def get_paths_from_file(filename): - new_path_changes = [] - with open(filename, 'br') as f: - for line in f: - line = line.rstrip(b'\r\n') - - # Skip blank lines - if not line: - continue - # Skip comment lines - if line.startswith(b'#'): - continue - - # Determine the replacement - match_type, repl = 'literal', None - if b'==>' in line: - line, repl = line.rsplit(b'==>', 1) - - # See if we need to match via regex - match_type = 'match' # a.k.a. 'literal' - if line.startswith(b'regex:'): - match_type = 'regex' - match = re.compile(line[6:]) - elif line.startswith(b'glob:'): - match_type = 'glob' - match = line[5:] - if repl: - raise SystemExit(_("Error: In %s, 'glob:' and '==>' are incompatible (renaming globs makes no sense)" % decode(filename))) - else: - if line.startswith(b'literal:'): - match = line[8:] - else: - match = line - if repl is not None: - if match and repl and match.endswith(b'/') != repl.endswith(b'/'): - raise SystemExit(_("Error: When rename directories, if OLDNAME " - "and NEW_NAME are both non-empty and either " - "ends with a slash then both must.")) - - # Record the filter or rename - if repl is not None: - new_path_changes.append(['rename', match_type, (match, repl)]) - else: - new_path_changes.append(['filter', match_type, match]) - if match_type == 'glob' and not match.endswith(b'*'): - extension = b'*' if match.endswith(b'/') else b'/*' - new_path_changes.append(['filter', match_type, match+extension]) - return new_path_changes - - @staticmethod - def default_options(): - return FilteringOptions.parse_args([], error_on_empty = False) - - @staticmethod - def parse_args(input_args, error_on_empty = True): - parser = FilteringOptions.create_arg_parser() - if not input_args and error_on_empty: - parser.print_usage() - raise SystemExit(_("No arguments specified.")) - args = parser.parse_args(input_args) - if args.help: - parser.print_help() - raise SystemExit() - if args.version: - GitUtils.print_my_version() - raise SystemExit() - FilteringOptions.sanity_check_args(args) - if args.mailmap: - args.mailmap = MailmapInfo(args.mailmap) - if args.replace_text: - args.replace_text = FilteringOptions.get_replace_text(args.replace_text) - if args.replace_message: - args.replace_message = FilteringOptions.get_replace_text(args.replace_message) - if args.strip_blobs_with_ids: - with open(args.strip_blobs_with_ids, 'br') as f: - args.strip_blobs_with_ids = set(f.read().split()) - else: - args.strip_blobs_with_ids = set() - if (args.partial or args.refs) and not args.replace_refs: - args.replace_refs = 'update-no-add' - args.repack = not (args.partial or args.refs) - if args.refs or args.source or args.target: - args.partial = True - if not args.refs: - args.refs = ['--all'] - return args - -class RepoAnalyze(object): - - # First, several helper functions for analyze_commit() - - @staticmethod - def equiv_class(stats, filename): - return stats['equivalence'].get(filename, (filename,)) - - @staticmethod - def setup_equivalence_for_rename(stats, oldname, newname): - # if A is renamed to B and B is renamed to C, then the user thinks of - # A, B, and C as all being different names for the same 'file'. We record - # this as an equivalence class: - # stats['equivalence'][name] = (A,B,C) - # for name being each of A, B, and C. - old_tuple = stats['equivalence'].get(oldname, ()) - if newname in old_tuple: - return - elif old_tuple: - new_tuple = tuple(list(old_tuple)+[newname]) - else: - new_tuple = (oldname, newname) - for f in new_tuple: - stats['equivalence'][f] = new_tuple - - @staticmethod - def setup_or_update_rename_history(stats, commit, oldname, newname): - rename_commits = stats['rename_history'].get(oldname, set()) - rename_commits.add(commit) - stats['rename_history'][oldname] = rename_commits - - @staticmethod - def handle_renames(stats, commit, change_types, filenames): - for index, change_type in enumerate(change_types): - if change_type == ord(b'R'): - oldname, newname = filenames[index], filenames[-1] - RepoAnalyze.setup_equivalence_for_rename(stats, oldname, newname) - RepoAnalyze.setup_or_update_rename_history(stats, commit, - oldname, newname) - - @staticmethod - def handle_file(stats, graph, commit, modes, shas, filenames): - mode, sha, filename = modes[-1], shas[-1], filenames[-1] - - # Figure out kind of deletions to undo for this file, and update lists - # of all-names-by-sha and all-filenames - delmode = 'tree_deletions' - if mode != b'040000': - delmode = 'file_deletions' - stats['names'][sha].add(filename) - stats['allnames'].add(filename) - - # If the file (or equivalence class of files) was recorded as deleted, - # clearly it isn't anymore - equiv = RepoAnalyze.equiv_class(stats, filename) - for f in equiv: - stats[delmode].pop(f, None) - - # If we get a modify/add for a path that was renamed, we may need to break - # the equivalence class. However, if the modify/add was on a branch that - # doesn't have the rename in its history, we are still okay. - need_to_break_equivalence = False - if equiv[-1] != filename: - for rename_commit in stats['rename_history'][filename]: - if graph.is_ancestor(rename_commit, commit): - need_to_break_equivalence = True - - if need_to_break_equivalence: - for f in equiv: - if f in stats['equivalence']: - del stats['equivalence'][f] - - @staticmethod - def analyze_commit(stats, graph, commit, parents, date, file_changes): - graph.add_commit_and_parents(commit, parents) - for change in file_changes: - modes, shas, change_types, filenames = change - if len(parents) == 1 and change_types.startswith(b'R'): - change_types = b'R' # remove the rename score; we don't care - if modes[-1] == b'160000': - continue - elif modes[-1] == b'000000': - # Track when files/directories are deleted - for f in RepoAnalyze.equiv_class(stats, filenames[-1]): - if any(x == b'040000' for x in modes[0:-1]): - stats['tree_deletions'][f] = date - else: - stats['file_deletions'][f] = date - elif change_types.strip(b'AMT') == b'': - RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames) - elif modes[-1] == b'040000' and change_types.strip(b'RAM') == b'': - RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames) - elif change_types.strip(b'RAMT') == b'': - RepoAnalyze.handle_file(stats, graph, commit, modes, shas, filenames) - RepoAnalyze.handle_renames(stats, commit, change_types, filenames) - else: - raise SystemExit(_("Unhandled change type(s): %(change_type)s " - "(in commit %(commit)s)") - % ({'change_type': change_types, 'commit': commit}) - ) # pragma: no cover - - @staticmethod - def gather_data(args): - unpacked_size, packed_size = GitUtils.get_blob_sizes() - stats = {'names': collections.defaultdict(set), - 'allnames' : set(), - 'file_deletions': {}, - 'tree_deletions': {}, - 'equivalence': {}, - 'rename_history': collections.defaultdict(set), - 'unpacked_size': unpacked_size, - 'packed_size': packed_size, - 'num_commits': 0} - - # Setup the rev-list/diff-tree process - processed_commits_msg = _("Processed %d commits") - commit_parse_progress = ProgressWriter() - num_commits = 0 - cmd = ('git rev-list --topo-order --reverse {}'.format(' '.join(args.refs)) + - ' | git diff-tree --stdin --always --root --format=%H%n%P%n%cd' + - ' --date=short -M -t -c --raw --combined-all-paths') - dtp = subproc.Popen(cmd, shell=True, bufsize=-1, stdout=subprocess.PIPE) - f = dtp.stdout - line = f.readline() - if not line: - raise SystemExit(_("Nothing to analyze; repository is empty.")) - cont = bool(line) - graph = AncestryGraph() - while cont: - commit = line.rstrip() - parents = f.readline().split() - date = f.readline().rstrip() - - # We expect a blank line next; if we get a non-blank line then - # this commit modified no files and we need to move on to the next. - # If there is no line, we've reached end-of-input. - line = f.readline() - if not line: - cont = False - line = line.rstrip() - - # If we haven't reached end of input, and we got a blank line meaning - # a commit that has modified files, then get the file changes associated - # with this commit. - file_changes = [] - if cont and not line: - cont = False - for line in f: - if not line.startswith(b':'): - cont = True - break - n = 1+max(1, len(parents)) - assert line.startswith(b':'*(n-1)) - relevant = line[n-1:-1] - splits = relevant.split(None, n) - modes = splits[0:n] - splits = splits[n].split(None, n) - shas = splits[0:n] - splits = splits[n].split(b'\t') - change_types = splits[0] - filenames = [PathQuoting.dequote(x) for x in splits[1:]] - file_changes.append([modes, shas, change_types, filenames]) - - # If someone is trying to analyze a subset of the history, make sure - # to avoid dying on commits with parents that we haven't seen before - if args.refs: - graph.record_external_commits([p for p in parents - if not p in graph.value]) - - # Analyze this commit and update progress - RepoAnalyze.analyze_commit(stats, graph, commit, parents, date, - file_changes) - num_commits += 1 - commit_parse_progress.show(processed_commits_msg % num_commits) - - # Show the final commits processed message and record the number of commits - commit_parse_progress.finish() - stats['num_commits'] = num_commits - - # Close the output, ensure rev-list|diff-tree pipeline completed successfully - dtp.stdout.close() - if dtp.wait(): - raise SystemExit(_("Error: rev-list|diff-tree pipeline failed; see above.")) # pragma: no cover - - return stats - - @staticmethod - def write_report(reportdir, stats): - def datestr(datetimestr): - return datetimestr if datetimestr else _('').encode() - - def dirnames(path): - while True: - path = os.path.dirname(path) - yield path - if path == b'': - break - - # Compute aggregate size information for paths, extensions, and dirs - total_size = {'packed': 0, 'unpacked': 0} - path_size = {'packed': collections.defaultdict(int), - 'unpacked': collections.defaultdict(int)} - ext_size = {'packed': collections.defaultdict(int), - 'unpacked': collections.defaultdict(int)} - dir_size = {'packed': collections.defaultdict(int), - 'unpacked': collections.defaultdict(int)} - for sha in stats['names']: - size = {'packed': stats['packed_size'][sha], - 'unpacked': stats['unpacked_size'][sha]} - for which in ('packed', 'unpacked'): - for name in stats['names'][sha]: - total_size[which] += size[which] - path_size[which][name] += size[which] - basename, ext = os.path.splitext(name) - ext_size[which][ext] += size[which] - for dirname in dirnames(name): - dir_size[which][dirname] += size[which] - - # Determine if and when extensions and directories were deleted - ext_deleted_data = {} - for name in stats['allnames']: - when = stats['file_deletions'].get(name, None) - - # Update the extension - basename, ext = os.path.splitext(name) - if when is None: - ext_deleted_data[ext] = None - elif ext in ext_deleted_data: - if ext_deleted_data[ext] is not None: - ext_deleted_data[ext] = max(ext_deleted_data[ext], when) - else: - ext_deleted_data[ext] = when - - dir_deleted_data = {} - for name in dir_size['packed']: - dir_deleted_data[name] = stats['tree_deletions'].get(name, None) - - with open(os.path.join(reportdir, b"README"), 'bw') as f: - # Give a basic overview of this file - f.write(b"== %s ==\n" % _("Overall Statistics").encode()) - f.write((" %s: %d\n" % (_("Number of commits"), - stats['num_commits'])).encode()) - f.write((" %s: %d\n" % (_("Number of filenames"), - len(path_size['packed']))).encode()) - f.write((" %s: %d\n" % (_("Number of directories"), - len(dir_size['packed']))).encode()) - f.write((" %s: %d\n" % (_("Number of file extensions"), - len(ext_size['packed']))).encode()) - f.write(b"\n") - f.write((" %s: %d\n" % (_("Total unpacked size (bytes)"), - total_size['unpacked'])).encode()) - f.write((" %s: %d\n" % (_("Total packed size (bytes)"), - total_size['packed'])).encode()) - f.write(b"\n") - - # Mention issues with the report - f.write(("== %s ==\n" % _("Caveats")).encode()) - f.write(("=== %s ===\n" % _("Sizes")).encode()) - f.write(textwrap.dedent(_(""" - Packed size represents what size your repository would be if no - trees, commits, tags, or other metadata were included (though it may - fail to represent de-duplication; see below). It also represents the - current packing, which may be suboptimal if you haven't gc'ed for a - while. - - Unpacked size represents what size your repository would be if no - trees, commits, tags, or other metadata were included AND if no - files were packed; i.e., without delta-ing or compression. - - Both unpacked and packed sizes can be slightly misleading. Deleting - a blob from history not save as much space as the unpacked size, - because it is obviously normally stored in packed form. Also, - deleting a blob from history may not save as much space as its packed - size either, because another blob could be stored as a delta against - that blob, so when you remove one blob another blob's packed size may - grow. - - Also, the sum of the packed sizes can add up to more than the - repository size; if the same contents appeared in the repository in - multiple places, git will automatically de-dupe and store only one - copy, while the way sizes are added in this analysis adds the size - for each file path that has those contents. Further, if a file is - ever reverted to a previous version's contents, the previous - version's size will be counted multiple times in this analysis, even - though git will only store it once. - """)[1:]).encode()) - f.write(b"\n") - f.write(("=== %s ===\n" % _("Deletions")).encode()) - f.write(textwrap.dedent(_(""" - Whether a file is deleted is not a binary quality, since it can be - deleted on some branches but still exist in others. Also, it might - exist in an old tag, but have been deleted in versions newer than - that. More thorough tracking could be done, including looking at - merge commits where one side of history deleted and the other modified, - in order to give a more holistic picture of deletions. However, that - algorithm would not only be more complex to implement, it'd also be - quite difficult to present and interpret by users. Since --analyze - is just about getting a high-level rough picture of history, it instead - implements the simplistic rule that is good enough for 98% of cases: - A file is marked as deleted if the last commit in the fast-export - stream that mentions the file lists it as deleted. - This makes it dependent on topological ordering, but generally gives - the "right" answer. - """)[1:]).encode()) - f.write(b"\n") - f.write(("=== %s ===\n" % _("Renames")).encode()) - f.write(textwrap.dedent(_(""" - Renames share the same non-binary nature that deletions do, plus - additional challenges: - * If the renamed file is renamed again, instead of just two names for - a path you can have three or more. - * Rename pairs of the form (oldname, newname) that we consider to be - different names of the "same file" might only be valid over certain - commit ranges. For example, if a new commit reintroduces a file - named oldname, then new versions of oldname aren't the "same file" - anymore. We could try to portray this to the user, but it's easier - for the user to just break the pairing and only report unbroken - rename pairings to the user. - * The ability for users to rename files differently in different - branches means that our chains of renames will not necessarily be - linear but may branch out. - """)[1:]).encode()) - f.write(b"\n") - - # Equivalence classes for names, so if folks only want to keep a - # certain set of paths, they know the old names they want to include - # too. - with open(os.path.join(reportdir, b"renames.txt"), 'bw') as f: - seen = set() - for pathname,equiv_group in sorted(stats['equivalence'].items(), - key=lambda x:(x[1], x[0])): - if equiv_group in seen: - continue - seen.add(equiv_group) - f.write(("{} ->\n ".format(decode(equiv_group[0])) + - "\n ".join(decode(x) for x in equiv_group[1:]) + - "\n").encode()) - - # List directories in reverse sorted order of unpacked size - with open(os.path.join(reportdir, b"directories-deleted-sizes.txt"), 'bw') as f: - msg = "=== %s ===\n" % _("Deleted directories by reverse size") - f.write(msg.encode()) - msg = _("Format: unpacked size, packed size, date deleted, directory name\n") - f.write(msg.encode()) - for dirname, size in sorted(dir_size['packed'].items(), - key=lambda x:(x[1],x[0]), reverse=True): - if (dir_deleted_data[dirname]): - f.write(b" %10d %10d %-10s %s\n" % (dir_size['unpacked'][dirname], - size, - datestr(dir_deleted_data[dirname]), - dirname or _('').encode())) - - with open(os.path.join(reportdir, b"directories-all-sizes.txt"), 'bw') as f: - f.write(("=== %s ===\n" % _("All directories by reverse size")).encode()) - msg = _("Format: unpacked size, packed size, date deleted, directory name\n") - f.write(msg.encode()) - for dirname, size in sorted(dir_size['packed'].items(), - key=lambda x:(x[1],x[0]), reverse=True): - f.write(b" %10d %10d %-10s %s\n" % (dir_size['unpacked'][dirname], - size, - datestr(dir_deleted_data[dirname]), - dirname or _("").encode())) - - # List extensions in reverse sorted order of unpacked size - with open(os.path.join(reportdir, b"extensions-deleted-sizes.txt"), 'bw') as f: - msg = "=== %s ===\n" % _("Deleted extensions by reverse size") - f.write(msg.encode()) - msg = _("Format: unpacked size, packed size, date deleted, extension name\n") - f.write(msg.encode()) - for extname, size in sorted(ext_size['packed'].items(), - key=lambda x:(x[1],x[0]), reverse=True): - if (ext_deleted_data[extname]): - f.write(b" %10d %10d %-10s %s\n" % (ext_size['unpacked'][extname], - size, - datestr(ext_deleted_data[extname]), - extname or _('').encode())) - - with open(os.path.join(reportdir, b"extensions-all-sizes.txt"), 'bw') as f: - f.write(("=== %s ===\n" % _("All extensions by reverse size")).encode()) - msg = _("Format: unpacked size, packed size, date deleted, extension name\n") - f.write(msg.encode()) - for extname, size in sorted(ext_size['packed'].items(), - key=lambda x:(x[1],x[0]), reverse=True): - f.write(b" %10d %10d %-10s %s\n" % (ext_size['unpacked'][extname], - size, - datestr(ext_deleted_data[extname]), - extname or _('').encode())) - - # List files in reverse sorted order of unpacked size - with open(os.path.join(reportdir, b"path-deleted-sizes.txt"), 'bw') as f: - msg = "=== %s ===\n" % _("Deleted paths by reverse accumulated size") - f.write(msg.encode()) - msg = _("Format: unpacked size, packed size, date deleted, path name(s)\n") - f.write(msg.encode()) - for pathname, size in sorted(path_size['packed'].items(), - key=lambda x:(x[1],x[0]), reverse=True): - when = stats['file_deletions'].get(pathname, None) - if when: - f.write(b" %10d %10d %-10s %s\n" % (path_size['unpacked'][pathname], - size, - datestr(when), - pathname)) - - with open(os.path.join(reportdir, b"path-all-sizes.txt"), 'bw') as f: - msg = "=== %s ===\n" % _("All paths by reverse accumulated size") - f.write(msg.encode()) - msg = _("Format: unpacked size, packed size, date deleted, path name\n") - f.write(msg.encode()) - for pathname, size in sorted(path_size['packed'].items(), - key=lambda x:(x[1],x[0]), reverse=True): - when = stats['file_deletions'].get(pathname, None) - f.write(b" %10d %10d %-10s %s\n" % (path_size['unpacked'][pathname], - size, - datestr(when), - pathname)) - - # List of filenames and sizes in descending order - with open(os.path.join(reportdir, b"blob-shas-and-paths.txt"), 'bw') as f: - f.write(("=== %s ===\n" % _("Files by sha and associated pathnames in reverse size")).encode()) - f.write(_("Format: sha, unpacked size, packed size, filename(s) object stored as\n").encode()) - for sha, size in sorted(stats['packed_size'].items(), - key=lambda x:(x[1],x[0]), reverse=True): - if sha not in stats['names']: - # Some objects in the repository might not be referenced, or not - # referenced by the branches/tags the user cares about; skip them. - continue - names_with_sha = stats['names'][sha] - if len(names_with_sha) == 1: - names_with_sha = names_with_sha.pop() - else: - names_with_sha = b'[' + b', '.join(sorted(names_with_sha)) + b']' - f.write(b" %s %10d %10d %s\n" % (sha, - stats['unpacked_size'][sha], - size, - names_with_sha)) - - @staticmethod - def run(args): - if args.report_dir: - reportdir = args.report_dir - else: - git_dir = GitUtils.determine_git_dir(b'.') - - # Create the report directory as necessary - results_tmp_dir = os.path.join(git_dir, b'filter-repo') - if not os.path.isdir(results_tmp_dir): - os.mkdir(results_tmp_dir) - reportdir = os.path.join(results_tmp_dir, b"analysis") - - if os.path.isdir(reportdir): - if args.force: - sys.stdout.write(_("Warning: Removing recursively: \"%s\"") % decode(reportdir)) - shutil.rmtree(reportdir) - else: - sys.stdout.write(_("Error: dir already exists (use --force to delete): \"%s\"\n") % decode(reportdir)) - sys.exit(1) - - os.mkdir(reportdir) - - # Gather the data we need - stats = RepoAnalyze.gather_data(args) - - # Write the reports - sys.stdout.write(_("Writing reports to %s...") % decode(reportdir)) - sys.stdout.flush() - RepoAnalyze.write_report(reportdir, stats) - sys.stdout.write(_("done.\n")) - -class InputFileBackup: - def __init__(self, input_file, output_file): - self.input_file = input_file - self.output_file = output_file - - def close(self): - self.input_file.close() - self.output_file.close() - - def read(self, size): - output = self.input_file.read(size) - self.output_file.write(output) - return output - - def readline(self): - line = self.input_file.readline() - self.output_file.write(line) - return line - -class DualFileWriter: - def __init__(self, file1, file2): - self.file1 = file1 - self.file2 = file2 - - def write(self, *args): - self.file1.write(*args) - self.file2.write(*args) - - def flush(self): - self.file1.flush() - self.file2.flush() - - def close(self): - self.file1.close() - self.file2.close() - -class RepoFilter(object): - def __init__(self, - args, - filename_callback = None, - message_callback = None, - name_callback = None, - email_callback = None, - refname_callback = None, - blob_callback = None, - commit_callback = None, - tag_callback = None, - reset_callback = None, - done_callback = None): - - self._args = args - - # Repo we are exporting - self._repo_working_dir = None - - # Store callbacks for acting on objects printed by FastExport - self._blob_callback = blob_callback - self._commit_callback = commit_callback - self._tag_callback = tag_callback - self._reset_callback = reset_callback - self._done_callback = done_callback - - # Store callbacks for acting on slices of FastExport objects - self._filename_callback = filename_callback # filenames from commits - self._message_callback = message_callback # commit OR tag message - self._name_callback = name_callback # author, committer, tagger - self._email_callback = email_callback # author, committer, tagger - self._refname_callback = refname_callback # from commit/tag/reset - self._handle_arg_callbacks() - - # Defaults for input - self._input = None - self._fep = None # Fast Export Process - self._fe_orig = None # Path to where original fast-export output stored - self._fe_filt = None # Path to where filtered fast-export output stored - self._parser = None # FastExportParser object we are working with - - # Defaults for output - self._output = None - self._fip = None # Fast Import Process - self._import_pipes = None - self._managed_output = True - - # A tuple of (depth, list-of-ancestors). Commits and ancestors are - # identified by their id (their 'mark' in fast-export or fast-import - # speak). The depth of a commit is one more than the max depth of any - # of its ancestors. - self._graph = AncestryGraph() - # Another one, for ancestry of commits in the original repo - self._orig_graph = AncestryGraph() - - # Names of files that were tweaked in any commit; such paths could lead - # to subsequent commits being empty - self._files_tweaked = set() - - # A set of commit hash pairs (oldhash, newhash) which used to be merge - # commits but due to filtering were turned into non-merge commits. - # The commits probably have suboptimal commit messages (e.g. "Merge branch - # next into master"). - self._commits_no_longer_merges = [] - - # A dict of original_ids to new_ids; filtering commits means getting - # new commit hash (sha1sums), and we record the mapping both for - # diagnostic purposes and so we can rewrite commit messages. Note that - # the new_id can be None rather than a commit hash if the original - # commit became empty and was pruned or was otherwise dropped. - self._commit_renames = {} - - # A set of original_ids for which we have not yet gotten the - # new_ids; we use OrderedDict because we need to know the order of - # insertion, but the values are always ignored (and set to None). - # If there was an OrderedSet class, I'd use it instead. - self._pending_renames = collections.OrderedDict() - - # A dict of commit_hash[0:7] -> set(commit_hashes with that prefix). - # - # It's common for commit messages to refer to commits by abbreviated - # commit hashes, as short as 7 characters. To facilitate translating - # such short hashes, we have a mapping of prefixes to full old hashes. - self._commit_short_old_hashes = collections.defaultdict(set) - - # A set of commit hash references appearing in commit messages which - # mapped to a valid commit that was removed entirely in the filtering - # process. The commit message will continue to reference the - # now-missing commit hash, since there was nothing to map it to. - self._commits_referenced_but_removed = set() - - # Progress handling (number of commits parsed, etc.) - self._progress_writer = ProgressWriter() - self._num_commits = 0 - - # Size of blobs in the repo - self._unpacked_size = {} - - # Other vars - self._sanity_checks_handled = False - self._finalize_handled = False - self._orig_refs = None - self._newnames = {} - - # Cache a few message translations for performance reasons - self._parsed_message = _("Parsed %d commits") - - # Compile some regexes and cache those - self._hash_re = re.compile(br'(\b[0-9a-f]{7,40}\b)') - - def _handle_arg_callbacks(self): - def make_callback(argname, str): - exec('def callback({}, _do_not_use_this_var = None):\n'.format(argname)+ - ' '+'\n '.join(str.splitlines()), globals()) - return callback #namespace['callback'] - def handle(type): - callback_field = '_{}_callback'.format(type) - code_string = getattr(self._args, type+'_callback') - if code_string: - if os.path.exists(code_string): - with open(code_string, 'r', encoding='utf-8') as f: - code_string = f.read() - if getattr(self, callback_field): - raise SystemExit(_("Error: Cannot pass a %s_callback to RepoFilter " - "AND pass --%s-callback" - % (type, type))) - if 'return ' not in code_string and \ - type not in ('blob', 'commit', 'tag', 'reset'): - raise SystemExit(_("Error: --%s-callback should have a return statement") - % type) - setattr(self, callback_field, make_callback(type, code_string)) - handle('filename') - handle('message') - handle('name') - handle('email') - handle('refname') - handle('blob') - handle('commit') - handle('tag') - handle('reset') - - def _run_sanity_checks(self): - self._sanity_checks_handled = True - if not self._managed_output: - if not self._args.replace_refs: - # If not _managed_output we don't want to make extra changes to the - # repo, so set default to no-op 'update-no-add' - self._args.replace_refs = 'update-no-add' - return - - if self._args.debug: - print("[DEBUG] Passed arguments:\n{}".format(self._args)) - - # Determine basic repository information - target_working_dir = self._args.target or b'.' - self._orig_refs = GitUtils.get_refs(target_working_dir) - is_bare = GitUtils.is_repository_bare(target_working_dir) - - # Determine if this is second or later run of filter-repo - tmp_dir = self.results_tmp_dir(create_if_missing=False) - already_ran = os.path.isfile(os.path.join(tmp_dir, b'already_ran')) - - # Default for --replace-refs - if not self._args.replace_refs: - self._args.replace_refs = ('update-or-add' if already_ran - else 'update-and-add') - - # Do sanity checks from the correct directory - if not self._args.force and not already_ran: - cwd = os.getcwd() - os.chdir(target_working_dir) - RepoFilter.sanity_check(self._orig_refs, is_bare) - os.chdir(cwd) - - @staticmethod - def sanity_check(refs, is_bare): - def abort(reason): - try: - cmd = 'git config remote.origin.url' - output = subproc.check_output(cmd.split()).strip() - except subprocess.CalledProcessError as e: - output = None - msg = "" - if output and os.path.isdir(output): - msg = _("Note: when cloning local repositories, you need to pass\n" - " --no-local to git clone to avoid this issue.\n") - raise SystemExit( - _("Aborting: Refusing to destructively overwrite repo history since\n" - "this does not look like a fresh clone.\n" - " (%s)\n%s" - "Please operate on a fresh clone instead. If you want to proceed\n" - "anyway, use --force.") % (reason, msg)) - - # Make sure repo is fully packed, just like a fresh clone would be. - # Note that transfer.unpackLimit defaults to 100, meaning that a - # repository with no packs and less than 100 objects should be considered - # fully packed. - output = subproc.check_output('git count-objects -v'.split()) - stats = dict(x.split(b': ') for x in output.splitlines()) - num_packs = int(stats[b'packs']) - num_loose_objects = int(stats[b'count']) - if num_packs > 1 or \ - (num_packs == 1 and num_loose_objects > 0) or \ - num_loose_objects >= 100: - abort(_("expected freshly packed repo")) - - # Make sure there is precisely one remote, named "origin"...or that this - # is a new bare repo with no packs and no remotes - output = subproc.check_output('git remote'.split()).strip() - if not (output == b"origin" or (num_packs == 0 and not output)): - abort(_("expected one remote, origin")) - - # Avoid letting people running with weird setups and overwriting GIT_DIR - # elsewhere - git_dir = GitUtils.determine_git_dir(b'.') - if is_bare and git_dir != b'.': - abort(_("GIT_DIR must be .")) - elif not is_bare and git_dir != b'.git': - abort(_("GIT_DIR must be .git")) - - # Make sure that all reflogs have precisely one entry - reflog_dir=os.path.join(git_dir, b'logs') - for root, dirs, files in os.walk(reflog_dir): - for filename in files: - pathname = os.path.join(root, filename) - with open(pathname, 'br') as f: - if len(f.read().splitlines()) > 1: - shortpath = pathname[len(reflog_dir)+1:] - abort(_("expected at most one entry in the reflog for %s") % - decode(shortpath)) - - # Make sure there are no stashed changes - if b'refs/stash' in refs: - abort(_("has stashed changes")) - - # Do extra checks in non-bare repos - if not is_bare: - # Avoid uncommitted, unstaged, or untracked changes - if subproc.call('git diff --staged --quiet'.split()): - abort(_("you have uncommitted changes")) - if subproc.call('git diff --quiet'.split()): - abort(_("you have unstaged changes")) - if len(subproc.check_output('git ls-files -o'.split())) > 0: - abort(_("you have untracked changes")) - - # Avoid unpushed changes - for refname, rev in refs.items(): - if not refname.startswith(b'refs/heads/'): - continue - origin_ref = refname.replace(b'refs/heads/', b'refs/remotes/origin/') - if origin_ref not in refs: - abort(_('%s exists, but %s not found') % (decode(refname), - decode(origin_ref))) - if rev != refs[origin_ref]: - abort(_('%s does not match %s') % (decode(refname), - decode(origin_ref))) - - # Make sure there is only one worktree - output = subproc.check_output('git worktree list'.split()) - if len(output.splitlines()) > 1: - abort(_('you have multiple worktrees')) - - @staticmethod - def cleanup(repo, repack, reset, run_quietly=False, show_debuginfo=False): - ''' cleanup repo; if repack then expire reflogs and do a gc --prune=now. - if reset then do a reset --hard. Optionally also curb output if - run_quietly is True, or go the opposite direction and show extra - output if show_debuginfo is True. ''' - assert not (run_quietly and show_debuginfo) - - if (repack and not run_quietly and not show_debuginfo): - print(_("Repacking your repo and cleaning out old unneeded objects")) - quiet_flags = '--quiet' if run_quietly else '' - cleanup_cmds = [] - if repack: - cleanup_cmds = ['git reflog expire --expire=now --all'.split(), - 'git gc {} --prune=now'.format(quiet_flags).split()] - if reset: - cleanup_cmds.insert(0, 'git reset {} --hard'.format(quiet_flags).split()) - location_info = ' (in {})'.format(decode(repo)) if repo != b'.' else '' - for cmd in cleanup_cmds: - if show_debuginfo: - print("[DEBUG] Running{}: {}".format(location_info, ' '.join(cmd))) - subproc.call(cmd, cwd=repo) - - def _get_rename(self, old_hash): - # If we already know the rename, just return it - new_hash = self._commit_renames.get(old_hash, None) - if new_hash: - return new_hash - - # If it's not in the remaining pending renames, we don't know it - if old_hash is not None and old_hash not in self._pending_renames: - return None - - # Read through the pending renames until we find it or we've read them all, - # and return whatever we might find - self._flush_renames(old_hash) - return self._commit_renames.get(old_hash, None) - - def _flush_renames(self, old_hash=None, limit=0): - # Parse through self._pending_renames until we have read enough. We have - # read enough if: - # self._pending_renames is empty - # old_hash != None and we found a rename for old_hash - # limit > 0 and len(self._pending_renames) started less than 2*limit - # limit > 0 and len(self._pending_renames) < limit - if limit and len(self._pending_renames) < 2 * limit: - return - fi_input, fi_output = self._import_pipes - while self._pending_renames: - orig_id, ignore = self._pending_renames.popitem(last=False) - new_id = fi_output.readline().rstrip() - self._commit_renames[orig_id] = new_id - if old_hash == orig_id: - return - if limit and len(self._pending_renames) < limit: - return - - def _translate_commit_hash(self, matchobj_or_oldhash): - old_hash = matchobj_or_oldhash - if not isinstance(matchobj_or_oldhash, bytes): - old_hash = matchobj_or_oldhash.group(1) - orig_len = len(old_hash) - new_hash = self._get_rename(old_hash) - if new_hash is None: - if old_hash[0:7] not in self._commit_short_old_hashes: - self._commits_referenced_but_removed.add(old_hash) - return old_hash - possibilities = self._commit_short_old_hashes[old_hash[0:7]] - matches = [x for x in possibilities - if x[0:orig_len] == old_hash] - if len(matches) != 1: - self._commits_referenced_but_removed.add(old_hash) - return old_hash - old_hash = matches[0] - new_hash = self._get_rename(old_hash) - - assert new_hash is not None - return new_hash[0:orig_len] - - def _trim_extra_parents(self, orig_parents, parents): - '''Due to pruning of empty commits, some parents could be non-existent - (None) or otherwise redundant. Remove the non-existent parents, and - remove redundant parents so long as that doesn't transform a merge - commit into a non-merge commit. - - Returns a tuple: - (parents, new_first_parent_if_would_become_non_merge)''' - - always_prune = (self._args.prune_degenerate == 'always') - - # Pruning of empty commits means multiple things: - # * An original parent of this commit may have been pruned causing the - # need to rewrite the reported parent to the nearest ancestor. We - # want to know when we're dealing with such a parent. - # * Further, there may be no "nearest ancestor" if the entire history - # of that parent was also pruned. (Detectable by the parent being - # 'None') - # Remove all parents rewritten to None, and keep track of which parents - # were rewritten to an ancestor. - tmp = zip(parents, - orig_parents, - [(x in _SKIPPED_COMMITS or always_prune) for x in orig_parents]) - tmp2 = [x for x in tmp if x[0] is not None] - if not tmp2: - # All ancestors have been pruned; we have no parents. - return [], None - parents, orig_parents, is_rewritten = [list(x) for x in zip(*tmp2)] - - # We can't have redundant parents if we don't have at least 2 parents - if len(parents) < 2: - return parents, None - - # Don't remove redundant parents if user doesn't want us to - if self._args.prune_degenerate == 'never': - return parents, None - - # Remove duplicate parents (if both sides of history have lots of commits - # which become empty due to pruning, the most recent ancestor on both - # sides may be the same commit), except only remove parents that have - # been rewritten due to previous empty pruning. - seen = set() - seen_add = seen.add - # Deleting duplicate rewritten parents means keeping parents if either - # they have not been seen or they are ones that have not been rewritten. - parents_copy = parents - uniq = [[p, orig_parents[i], is_rewritten[i]] for i, p in enumerate(parents) - if not (p in seen or seen_add(p)) or not is_rewritten[i]] - parents, orig_parents, is_rewritten = [list(x) for x in zip(*uniq)] - if len(parents) < 2: - return parents_copy, parents[0] - - # Flatten unnecessary merges. (If one side of history is entirely - # empty commits that were pruned, we may end up attempting to - # merge a commit with its ancestor. Remove parents that are an - # ancestor of another parent.) - num_parents = len(parents) - to_remove = [] - for cur in range(num_parents): - if not is_rewritten[cur]: - continue - for other in range(num_parents): - if cur == other: - continue - if not self._graph.is_ancestor(parents[cur], parents[other]): - continue - # parents[cur] is an ancestor of parents[other], so parents[cur] - # seems redundant. However, if it was intentionally redundant - # (e.g. a no-ff merge) in the original, then we want to keep it. - if not always_prune and \ - self._orig_graph.is_ancestor(orig_parents[cur], - orig_parents[other]): - continue - # Some folks want their history to have all first parents be merge - # commits (except for any root commits), and always do a merge --no-ff. - # For such folks, don't remove the first parent even if it's an - # ancestor of other commits. - if self._args.no_ff and cur == 0: - continue - # Okay so the cur-th parent is an ancestor of the other-th parent, - # and it wasn't that way in the original repository; mark the - # cur-th parent as removable. - to_remove.append(cur) - break # cur removed, so skip rest of others -- i.e. check cur+=1 - for x in reversed(to_remove): - parents.pop(x) - if len(parents) < 2: - return parents_copy, parents[0] - - return parents, None - - def _prunable(self, commit, new_1st_parent, had_file_changes, orig_parents): - parents = commit.parents - - if self._args.prune_empty == 'never': - return False - always_prune = (self._args.prune_empty == 'always') - - # For merge commits, unless there are prunable (redundant) parents, we - # do not want to prune - if len(parents) >= 2 and not new_1st_parent: - return False - - if len(parents) < 2: - # Special logic for commits that started empty... - if not had_file_changes and not always_prune: - had_parents_pruned = (len(parents) < len(orig_parents) or - (len(orig_parents) == 1 and - orig_parents[0] in _SKIPPED_COMMITS)) - # If the commit remains empty and had parents which were pruned, - # then prune this commit; otherwise, retain it - return (not commit.file_changes and had_parents_pruned) - - # We can only get here if the commit didn't start empty, so if it's - # empty now, it obviously became empty - if not commit.file_changes: - return True - - # If there are no parents of this commit and we didn't match the case - # above, then this commit cannot be pruned. Since we have no parent(s) - # to compare to, abort now to prevent future checks from failing. - if not parents: - return False - - # Similarly, we cannot handle the hard cases if we don't have a pipe - # to communicate with fast-import - if not self._import_pipes: - return False - - # If there have not been renames/remappings of IDs (due to insertion of - # new blobs), then we can sometimes know things aren't prunable with a - # simple check - if not _IDS.has_renames(): - # non-merge commits can only be empty if blob/file-change editing caused - # all file changes in the commit to have the same file contents as - # the parent. - changed_files = set(change.filename for change in commit.file_changes) - if len(orig_parents) < 2 and changed_files - self._files_tweaked: - return False - - # Finally, the hard case: due to either blob rewriting, or due to pruning - # of empty commits wiping out the first parent history back to the merge - # base, the list of file_changes we have may not actually differ from our - # (new) first parent's version of the files, i.e. this would actually be - # an empty commit. Check by comparing the contents of this commit to its - # (remaining) parent. - # - # NOTE on why this works, for the case of original first parent history - # having been pruned away due to being empty: - # The first parent history having been pruned away due to being - # empty implies the original first parent would have a tree (after - # filtering) that matched the merge base's tree. Since - # file_changes has the changes needed to go from what would have - # been the first parent to our new commit, and what would have been - # our first parent has a tree that matches the merge base, then if - # the new first parent has a tree matching the versions of files in - # file_changes, then this new commit is empty and thus prunable. - fi_input, fi_output = self._import_pipes - self._flush_renames() # Avoid fi_output having other stuff present - # Optimization note: we could have two loops over file_changes, the - # first doing all the self._output.write() calls, and the second doing - # the rest. But I'm worried about fast-import blocking on fi_output - # buffers filling up so I instead read from it as I go. - for change in commit.file_changes: - parent = new_1st_parent or commit.parents[0] # exists due to above checks - quoted_filename = PathQuoting.enquote(change.filename) - if isinstance(parent, int): - self._output.write(b"ls :%d %s\n" % (parent, quoted_filename)) - else: - self._output.write(b"ls %s %s\n" % (parent, quoted_filename)) - self._output.flush() - parent_version = fi_output.readline().split() - if change.type == b'D': - if parent_version != [b'missing', quoted_filename]: - return False - else: - blob_sha = change.blob_id - if isinstance(change.blob_id, int): - self._output.write(b"get-mark :%d\n" % change.blob_id) - self._output.flush() - blob_sha = fi_output.readline().rstrip() - if parent_version != [change.mode, b'blob', blob_sha, quoted_filename]: - return False - - return True - - def _record_remapping(self, commit, orig_parents): - new_id = None - # Record the mapping of old commit hash to new one - if commit.original_id and self._import_pipes: - fi_input, fi_output = self._import_pipes - self._output.write(b"get-mark :%d\n" % commit.id) - self._output.flush() - orig_id = commit.original_id - self._commit_short_old_hashes[orig_id[0:7]].add(orig_id) - # Note that we have queued up an id for later reading; flush a - # few of the older ones if we have too many queued up - self._pending_renames[orig_id] = None - self._flush_renames(None, limit=40) - # Also, record if this was a merge commit that turned into a non-merge - # commit. - if len(orig_parents) >= 2 and len(commit.parents) < 2: - self._commits_no_longer_merges.append((commit.original_id, new_id)) - - def callback_metadata(self, extra_items = dict()): - return {'commit_rename_func': self._translate_commit_hash, - 'ancestry_graph': self._graph, - 'original_ancestry_graph': self._orig_graph, - **extra_items} - - def _tweak_blob(self, blob): - if self._args.max_blob_size and len(blob.data) > self._args.max_blob_size: - blob.skip() - - if blob.original_id in self._args.strip_blobs_with_ids: - blob.skip() - - if ( self._args.replace_text - # not (if blob contains zero byte in the first 8Kb, that is, if blob is binary data) - and not b"\0" in blob.data[0:8192] - ): - for literal, replacement in self._args.replace_text['literals']: - blob.data = blob.data.replace(literal, replacement) - for regex, replacement in self._args.replace_text['regexes']: - blob.data = regex.sub(replacement, blob.data) - - if self._blob_callback: - self._blob_callback(blob, self.callback_metadata()) - - def _filter_files(self, commit): - def filename_matches(path_expression, pathname): - ''' Returns whether path_expression matches pathname or a leading - directory thereof, allowing path_expression to not have a trailing - slash even if it is meant to match a leading directory. ''' - if path_expression == b'': - return True - n = len(path_expression) - if (pathname.startswith(path_expression) and - (path_expression[n-1:n] == b'/' or - len(pathname) == n or - pathname[n:n+1] == b'/')): - return True - return False - - def newname(path_changes, pathname, use_base_name, filtering_is_inclusive): - ''' Applies filtering and rename changes from path_changes to pathname, - returning any of None (file isn't wanted), original filename (file - is wanted with original name), or new filename. ''' - wanted = False - full_pathname = pathname - if use_base_name: - pathname = os.path.basename(pathname) - for (mod_type, match_type, path_exp) in path_changes: - if mod_type == 'filter' and not wanted: - assert match_type in ('match', 'glob', 'regex') - if match_type == 'match' and filename_matches(path_exp, pathname): - wanted = True - if match_type == 'glob' and fnmatch.fnmatch(pathname, path_exp): - wanted = True - if match_type == 'regex' and path_exp.search(pathname): - wanted = True - elif mod_type == 'rename': - match, repl = path_exp - assert match_type in ('match','regex') # glob was translated to regex - if match_type == 'match' and filename_matches(match, full_pathname): - full_pathname = full_pathname.replace(match, repl, 1) - if match_type == 'regex': - full_pathname = match.sub(repl, full_pathname) - return full_pathname if (wanted == filtering_is_inclusive) else None - - args = self._args - new_file_changes = {} # Assumes no renames or copies, otherwise collisions - for change in commit.file_changes: - # NEEDSWORK: _If_ we ever want to pass `--full-tree` to fast-export and - # parse that output, we'll need to modify this block; `--full-tree` - # issues a deleteall directive which has no filename, and thus this - # block would normally strip it. Of course, FileChange() and - # _parse_optional_filechange() would need updates too. - if change.type == b'DELETEALL': - new_file_changes[b''] = change - continue - if change.filename in self._newnames: - change.filename = self._newnames[change.filename] - else: - original_filename = change.filename - change.filename = newname(args.path_changes, change.filename, - args.use_base_name, args.inclusive) - if self._filename_callback: - change.filename = self._filename_callback(change.filename) - self._newnames[original_filename] = change.filename - if not change.filename: - continue # Filtering criteria excluded this file; move on to next one - if change.filename in new_file_changes: - # Getting here means that path renaming is in effect, and caused one - # path to collide with another. That's usually bad, but can be okay - # under two circumstances: - # 1) Sometimes people have a file named OLDFILE in old revisions of - # history, and they rename to NEWFILE, and would like to rewrite - # history so that all revisions refer to it as NEWFILE. As such, - # we can allow a collision when (at least) one of the two paths - # is a deletion. Note that if OLDFILE and NEWFILE are unrelated - # this also allows the rewrite to continue, which makes sense - # since OLDFILE is no longer in the way. - # 2) If OLDFILE and NEWFILE are exactly equal, then writing them - # both to the same location poses no problem; we only need one - # file. (This could come up if someone copied a file in some - # commit, then later either deleted the file or kept it exactly - # in sync with the original with any changes, and then decides - # they want to rewrite history to only have one of the two files) - colliding_change = new_file_changes[change.filename] - if change.type == b'D': - # We can just throw this one away and keep the other - continue - elif change.type == b'M' and ( - change.mode == colliding_change.mode and - change.blob_id == colliding_change.blob_id): - # The two are identical, so we can throw this one away and keep other - continue - elif new_file_changes[change.filename].type != b'D': - raise SystemExit(_("File renaming caused colliding pathnames!\n") + - _(" Commit: {}\n").format(commit.original_id) + - _(" Filename: {}").format(change.filename)) - # Strip files that are too large - if self._args.max_blob_size and \ - self._unpacked_size.get(change.blob_id, 0) > self._args.max_blob_size: - continue - if self._args.strip_blobs_with_ids and \ - change.blob_id in self._args.strip_blobs_with_ids: - continue - # Otherwise, record the change - new_file_changes[change.filename] = change - commit.file_changes = [v for k,v in sorted(new_file_changes.items())] - - def _tweak_commit(self, commit, aux_info): - # Change the commit message according to callback - if not self._args.preserve_commit_hashes: - commit.message = self._hash_re.sub(self._translate_commit_hash, - commit.message) - if self._args.replace_message: - for literal, replacement in self._args.replace_message['literals']: - commit.message = commit.message.replace(literal, replacement) - for regex, replacement in self._args.replace_message['regexes']: - commit.message = regex.sub(replacement, commit.message) - if self._message_callback: - commit.message = self._message_callback(commit.message) - - # Change the author & committer according to mailmap rules - args = self._args - if args.mailmap: - commit.author_name, commit.author_email = \ - args.mailmap.translate(commit.author_name, commit.author_email) - commit.committer_name, commit.committer_email = \ - args.mailmap.translate(commit.committer_name, commit.committer_email) - # Change author & committer according to callbacks - if self._name_callback: - commit.author_name = self._name_callback(commit.author_name) - commit.committer_name = self._name_callback(commit.committer_name) - if self._email_callback: - commit.author_email = self._email_callback(commit.author_email) - commit.committer_email = self._email_callback(commit.committer_email) - - # Sometimes the 'branch' given is a tag; if so, rename it as requested so - # we don't get any old tagnames - if self._args.tag_rename: - commit.branch = RepoFilter._do_tag_rename(args.tag_rename, commit.branch) - if self._refname_callback: - commit.branch = self._refname_callback(commit.branch) - - # Filter or rename the list of file changes - orig_file_changes = set(commit.file_changes) - self._filter_files(commit) - - # Record ancestry graph - parents, orig_parents = commit.parents, aux_info['orig_parents'] - if self._args.state_branch: - external_parents = parents - else: - external_parents = [p for p in parents if not isinstance(p, int)] - self._graph.record_external_commits(external_parents) - self._orig_graph.record_external_commits(external_parents) - self._graph.add_commit_and_parents(commit.id, parents) - self._orig_graph.add_commit_and_parents(commit.old_id, orig_parents) - - # Prune parents (due to pruning of empty commits) if relevant - old_1st_parent = parents[0] if parents else None - parents, new_1st_parent = self._trim_extra_parents(orig_parents, parents) - commit.parents = parents - - # If parents were pruned, then we need our file changes to be relative - # to the new first parent - if parents and old_1st_parent != parents[0]: - commit.file_changes = GitUtils.get_file_changes(self._repo_working_dir, - ID_TO_HASH[parents[0]], - commit.original_id) - orig_file_changes = set(commit.file_changes) - self._filter_files(commit) - - # Find out which files were modified by the callbacks. Such paths could - # lead to subsequent commits being empty (e.g. if removing a line containing - # a password from every version of a file that had the password, and some - # later commit did nothing more than remove that line) - final_file_changes = set(commit.file_changes) - if self._args.replace_text or self._blob_callback: - differences = orig_file_changes.union(final_file_changes) - else: - differences = orig_file_changes.symmetric_difference(final_file_changes) - self._files_tweaked.update(x.filename for x in differences) - - # Call the user-defined callback, if any - if self._commit_callback: - self._commit_callback(commit, self.callback_metadata(aux_info)) - - # Now print the resulting commit, or if prunable skip it - if not commit.dumped: - if not self._prunable(commit, new_1st_parent, - aux_info['had_file_changes'], orig_parents): - self._insert_into_stream(commit) - self._record_remapping(commit, orig_parents) - else: - rewrite_to = new_1st_parent or commit.first_parent() - commit.skip(new_id = rewrite_to) - if self._args.state_branch: - alias = Alias(commit.old_id or commit.id, rewrite_to or deleted_hash) - self._insert_into_stream(alias) - reset = Reset(commit.branch, rewrite_to or deleted_hash) - self._insert_into_stream(reset) - self._commit_renames[commit.original_id] = None - - # Show progress - self._num_commits += 1 - if not self._args.quiet: - self._progress_writer.show(self._parsed_message % self._num_commits) - - @staticmethod - def _do_tag_rename(rename_pair, tagname): - old, new = rename_pair.split(b':', 1) - old, new = b'refs/tags/'+old, b'refs/tags/'+new - if tagname.startswith(old): - return tagname.replace(old, new, 1) - return tagname - - def _tweak_tag(self, tag): - # Tweak the tag message according to callbacks - if self._args.replace_message: - for literal, replacement in self._args.replace_message['literals']: - tag.message = tag.message.replace(literal, replacement) - for regex, replacement in self._args.replace_message['regexes']: - tag.message = regex.sub(replacement, tag.message) - if self._message_callback: - tag.message = self._message_callback(tag.message) - - # Tweak the tag name according to tag-name-related callbacks - tag_prefix = b'refs/tags/' - fullref = tag_prefix+tag.ref - if self._args.tag_rename: - fullref = RepoFilter._do_tag_rename(self._args.tag_rename, fullref) - if self._refname_callback: - fullref = self._refname_callback(fullref) - if not fullref.startswith(tag_prefix): - msg = "Error: fast-import requires tags to be in refs/tags/ namespace." - msg += "\n {} renamed to {}".format(tag_prefix+tag.ref, fullref) - raise SystemExit(msg) - tag.ref = fullref[len(tag_prefix):] - - # Tweak the tagger according to callbacks - if self._args.mailmap: - tag.tagger_name, tag.tagger_email = \ - self._args.mailmap.translate(tag.tagger_name, tag.tagger_email) - if self._name_callback: - tag.tagger_name = self._name_callback(tag.tagger_name) - if self._email_callback: - tag.tagger_email = self._email_callback(tag.tagger_email) - - # Call general purpose tag callback - if self._tag_callback: - self._tag_callback(tag, self.callback_metadata()) - - def _tweak_reset(self, reset): - if self._args.tag_rename: - reset.ref = RepoFilter._do_tag_rename(self._args.tag_rename, reset.ref) - if self._refname_callback: - reset.ref = self._refname_callback(reset.ref) - if self._reset_callback: - self._reset_callback(reset, self.callback_metadata()) - - def results_tmp_dir(self, create_if_missing=True): - target_working_dir = self._args.target or b'.' - git_dir = GitUtils.determine_git_dir(target_working_dir) - d = os.path.join(git_dir, b'filter-repo') - if create_if_missing and not os.path.isdir(d): - os.mkdir(d) - return d - - def _load_marks_file(self, marks_basename): - full_branch = 'refs/heads/{}'.format(self._args.state_branch) - marks_file = os.path.join(self.results_tmp_dir(), marks_basename) - working_dir = self._args.target or b'.' - cmd = ['git', '-C', working_dir, 'show-ref', full_branch] - contents = b'' - if subproc.call(cmd, stdout=subprocess.DEVNULL) == 0: - cmd = ['git', '-C', working_dir, 'show', - '%s:%s' % (full_branch, decode(marks_basename))] - try: - contents = subproc.check_output(cmd) - except subprocess.CalledProcessError as e: # pragma: no cover - raise SystemExit(_("Failed loading %s from %s") % - (decode(marks_basename), full_branch)) - if contents: - biggest_id = max(int(x.split()[0][1:]) for x in contents.splitlines()) - _IDS._next_id = max(_IDS._next_id, biggest_id+1) - with open(marks_file, 'bw') as f: - f.write(contents) - return marks_file - - def _save_marks_files(self): - basenames = [b'source-marks', b'target-marks'] - working_dir = self._args.target or b'.' - - # Check whether the branch exists - parent = [] - full_branch = 'refs/heads/{}'.format(self._args.state_branch) - cmd = ['git', '-C', working_dir, 'show-ref', full_branch] - if subproc.call(cmd, stdout=subprocess.DEVNULL) == 0: - parent = ['-p', full_branch] - - # Run 'git hash-object $MARKS_FILE' for each marks file, save result - blob_hashes = {} - for marks_basename in basenames: - marks_file = os.path.join(self.results_tmp_dir(), marks_basename) - if not os.path.isfile(marks_file): # pragma: no cover - raise SystemExit(_("Failed to find %s to save to %s") - % (marks_file, self._args.state_branch)) - cmd = ['git', '-C', working_dir, 'hash-object', '-w', marks_file] - blob_hashes[marks_basename] = subproc.check_output(cmd).strip() - - # Run 'git mktree' to create a tree out of it - p = subproc.Popen(['git', '-C', working_dir, 'mktree'], - stdin=subprocess.PIPE, stdout=subprocess.PIPE) - for b in basenames: - p.stdin.write(b'100644 blob %s\t%s\n' % (blob_hashes[b], b)) - p.stdin.close() - p.wait() - tree = p.stdout.read().strip() - - # Create the new commit - cmd = (['git', '-C', working_dir, 'commit-tree', '-m', 'New mark files', - tree] + parent) - commit = subproc.check_output(cmd).strip() - subproc.call(['git', '-C', working_dir, 'update-ref', full_branch, commit]) - - def importer_only(self): - self._run_sanity_checks() - self._setup_output() - - def set_output(self, outputRepoFilter): - assert outputRepoFilter._output - - # set_output implies this RepoFilter is doing exporting, though may not - # be the only one. - self._setup_input(use_done_feature = False) - - # Set our output management up to pipe to outputRepoFilter's locations - self._managed_output = False - self._output = outputRepoFilter._output - self._import_pipes = outputRepoFilter._import_pipes - - # Handle sanity checks, though currently none needed for export-only cases - self._run_sanity_checks() - - def _setup_input(self, use_done_feature): - if self._args.stdin: - self._input = sys.stdin.detach() - sys.stdin = None # Make sure no one tries to accidentally use it - self._fe_orig = None - else: - skip_blobs = (self._blob_callback is None and - self._args.replace_text is None and - self._args.source == self._args.target) - extra_flags = [] - if skip_blobs: - extra_flags.append('--no-data') - if self._args.max_blob_size: - self._unpacked_size, packed_size = GitUtils.get_blob_sizes() - if use_done_feature: - extra_flags.append('--use-done-feature') - if write_marks: - extra_flags.append(b'--mark-tags') - if self._args.state_branch: - assert(write_marks) - source_marks_file = self._load_marks_file(b'source-marks') - extra_flags.extend([b'--export-marks='+source_marks_file, - b'--import-marks='+source_marks_file]) - if self._args.preserve_commit_encoding is not None: # pragma: no cover - reencode = 'no' if self._args.preserve_commit_encoding else 'yes' - extra_flags.append('--reencode='+reencode) - location = ['-C', self._args.source] if self._args.source else [] - fep_cmd = ['git'] + location + ['fast-export', '--show-original-ids', - '--signed-tags=strip', '--tag-of-filtered-object=rewrite', - '--fake-missing-tagger', '--reference-excluded-parents' - ] + extra_flags + self._args.refs - self._fep = subproc.Popen(fep_cmd, bufsize=-1, stdout=subprocess.PIPE) - self._input = self._fep.stdout - if self._args.dry_run or self._args.debug: - self._fe_orig = os.path.join(self.results_tmp_dir(), - b'fast-export.original') - output = open(self._fe_orig, 'bw') - self._input = InputFileBackup(self._input, output) - if self._args.debug: - tmp = [decode(x) if isinstance(x, bytes) else x for x in fep_cmd] - print("[DEBUG] Running: {}".format(' '.join(tmp))) - print(" (saving a copy of the output at {})" - .format(decode(self._fe_orig))) - - def _setup_output(self): - if not self._args.dry_run: - location = ['-C', self._args.target] if self._args.target else [] - fip_cmd = ['git'] + location + ['-c', 'core.ignorecase=false', - 'fast-import', '--force', '--quiet'] - if date_format_permissive: - fip_cmd.append('--date-format=raw-permissive') - if self._args.state_branch: - target_marks_file = self._load_marks_file(b'target-marks') - fip_cmd.extend([b'--export-marks='+target_marks_file, - b'--import-marks='+target_marks_file]) - self._fip = subproc.Popen(fip_cmd, bufsize=-1, - stdin=subprocess.PIPE, stdout=subprocess.PIPE) - self._import_pipes = (self._fip.stdin, self._fip.stdout) - if self._args.dry_run or self._args.debug: - self._fe_filt = os.path.join(self.results_tmp_dir(), - b'fast-export.filtered') - self._output = open(self._fe_filt, 'bw') - else: - self._output = self._fip.stdin - if self._args.debug and not self._args.dry_run: - self._output = DualFileWriter(self._fip.stdin, self._output) - tmp = [decode(x) if isinstance(x, bytes) else x for x in fip_cmd] - print("[DEBUG] Running: {}".format(' '.join(tmp))) - print(" (using the following file as input: {})" - .format(decode(self._fe_filt))) - - def _migrate_origin_to_heads(self): - refs_to_migrate = set(x for x in self._orig_refs - if x.startswith(b'refs/remotes/origin/')) - if not refs_to_migrate: - return - if self._args.debug: - print("[DEBUG] Migrating refs/remotes/origin/* -> refs/heads/*") - target_working_dir = self._args.target or b'.' - p = subproc.Popen('git update-ref --no-deref --stdin'.split(), - stdin=subprocess.PIPE, cwd=target_working_dir) - for ref in refs_to_migrate: - if ref == b'refs/remotes/origin/HEAD': - p.stdin.write(b'delete %s %s\n' % (ref, self._orig_refs[ref])) - del self._orig_refs[ref] - continue - newref = ref.replace(b'refs/remotes/origin/', b'refs/heads/') - if newref not in self._orig_refs: - p.stdin.write(b'create %s %s\n' % (newref, self._orig_refs[ref])) - p.stdin.write(b'delete %s %s\n' % (ref, self._orig_refs[ref])) - self._orig_refs[newref] = self._orig_refs[ref] - del self._orig_refs[ref] - p.stdin.close() - if p.wait(): - raise SystemExit(_("git update-ref failed; see above")) # pragma: no cover - - # Now remove - if self._args.debug: - print("[DEBUG] Removing 'origin' remote (rewritten history will no ") - print(" longer be related; consider re-pushing it elsewhere.") - subproc.call('git remote rm origin'.split(), cwd=target_working_dir) - - def _final_commands(self): - self._finalize_handled = True - self._done_callback and self._done_callback() - - if not self._args.quiet: - self._progress_writer.finish() - - def _ref_update(self, target_working_dir): - # Start the update-ref process - p = subproc.Popen('git update-ref --no-deref --stdin'.split(), - stdin=subprocess.PIPE, - cwd=target_working_dir) - - # Remove replace_refs from _orig_refs - replace_refs = {k:v for k, v in self._orig_refs.items() - if k.startswith(b'refs/replace/')} - reverse_replace_refs = collections.defaultdict(list) - for k,v in replace_refs.items(): - reverse_replace_refs[v].append(k) - all(map(self._orig_refs.pop, replace_refs)) - - # Remove unused refs - exported_refs, imported_refs = self.get_exported_and_imported_refs() - refs_to_nuke = exported_refs - imported_refs - if self._args.partial: - refs_to_nuke = set() - if refs_to_nuke and self._args.debug: - print("[DEBUG] Deleting the following refs:\n "+ - decode(b"\n ".join(refs_to_nuke))) - p.stdin.write(b''.join([b"delete %s\n" % x - for x in refs_to_nuke])) - - # Delete or update and add replace_refs; note that fast-export automatically - # handles 'update-no-add', we only need to take action for the other four - # choices for replace_refs. - self._flush_renames() - actual_renames = {k:v for k,v in self._commit_renames.items() if k != v} - if self._args.replace_refs in ['delete-no-add', 'delete-and-add']: - # Delete old replace refs, if unwanted - replace_refs_to_nuke = set(replace_refs) - if self._args.replace_refs == 'delete-and-add': - # git-update-ref won't allow us to update a ref twice, so be careful - # to avoid deleting refs we'll later update - replace_refs_to_nuke = replace_refs_to_nuke.difference( - [b'refs/replace/'+x for x in actual_renames]) - p.stdin.write(b''.join([b"delete %s\n" % x - for x in replace_refs_to_nuke])) - if self._args.replace_refs in ['delete-and-add', 'update-or-add', - 'update-and-add']: - # Add new replace refs - update_only = (self._args.replace_refs == 'update-or-add') - p.stdin.write(b''.join([b"update refs/replace/%s %s\n" % (old, new) - for old,new in actual_renames.items() - if new and not (update_only and - old in reverse_replace_refs)])) - - # Complete the update-ref process - p.stdin.close() - if p.wait(): - raise SystemExit(_("git update-ref failed; see above")) # pragma: no cover - - def _record_metadata(self, metadata_dir, orig_refs): - self._flush_renames() - with open(os.path.join(metadata_dir, b'commit-map'), 'bw') as f: - f.write(("%-40s %s\n" % (_("old"), _("new"))).encode()) - for (old,new) in self._commit_renames.items(): - msg = b'%s %s\n' % (old, new if new != None else deleted_hash) - f.write(msg) - - exported_refs, imported_refs = self.get_exported_and_imported_refs() - - batch_check_process = None - batch_check_output_re = re.compile(b'^([0-9a-f]{40}) ([a-z]+) ([0-9]+)$') - with open(os.path.join(metadata_dir, b'ref-map'), 'bw') as f: - f.write(("%-40s %-40s %s\n" % (_("old"), _("new"), _("ref"))).encode()) - for refname, old_hash in orig_refs.items(): - if refname not in exported_refs: - continue - if refname not in imported_refs: - new_hash = deleted_hash - elif old_hash in self._commit_renames: - new_hash = self._commit_renames[old_hash] - new_hash = new_hash if new_hash != None else deleted_hash - else: # Must be either an annotated tag, or a ref whose tip was pruned - if not batch_check_process: - cmd = 'git cat-file --batch-check'.split() - target_working_dir = self._args.target or b'.' - batch_check_process = subproc.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - cwd=target_working_dir) - batch_check_process.stdin.write(refname+b"\n") - batch_check_process.stdin.flush() - line = batch_check_process.stdout.readline() - m = batch_check_output_re.match(line) - if m and m.group(2) in (b'tag', b'commit'): - new_hash = m.group(1) - elif line.endswith(b' missing\n'): - new_hash = deleted_hash - else: - raise SystemExit(_("Failed to find new id for %(refname)s " - "(old id was %(old_hash)s)") - % ({'refname': refname, 'old_hash': old_hash}) - ) # pragma: no cover - f.write(b'%s %s %s\n' % (old_hash, new_hash, refname)) - if self._args.source or self._args.target: - new_refs = GitUtils.get_refs(self._args.target or b'.') - for ref, new_hash in new_refs.items(): - if ref not in orig_refs and not ref.startswith(b'refs/replace/'): - old_hash = b'0'*len(new_hash) - f.write(b'%s %s %s\n' % (old_hash, new_hash, ref)) - if batch_check_process: - batch_check_process.stdin.close() - batch_check_process.wait() - - with open(os.path.join(metadata_dir, b'suboptimal-issues'), 'bw') as f: - issues_found = False - if self._commits_no_longer_merges: - issues_found = True - - f.write(textwrap.dedent(_(''' - The following commits used to be merge commits but due to filtering - are now regular commits; they likely have suboptimal commit messages - (e.g. "Merge branch next into master"). Original commit hash on the - left, commit hash after filtering/rewriting on the right: - ''')[1:]).encode()) - for oldhash, newhash in self._commits_no_longer_merges: - f.write(' {} {}\n'.format(oldhash, newhash).encode()) - f.write(b'\n') - - if self._commits_referenced_but_removed: - issues_found = True - f.write(textwrap.dedent(_(''' - The following commits were filtered out, but referenced in another - commit message. The reference to the now-nonexistent commit hash - (or a substring thereof) was left as-is in any commit messages: - ''')[1:]).encode()) - for bad_commit_reference in self._commits_referenced_but_removed: - f.write(' {}\n'.format(bad_commit_reference).encode()) - f.write(b'\n') - - if not issues_found: - f.write(_("No filtering problems encountered.\n").encode()) - - with open(os.path.join(metadata_dir, b'already_ran'), 'bw') as f: - f.write(_("This file exists to allow you to filter again without --force.\n").encode()) - - def finish(self): - ''' Alternative to run() when there is no input of our own to parse, - meaning that run only really needs to close the handle to fast-import - and let it finish, thus making a call to "run" feel like a misnomer. ''' - assert not self._input - assert self._managed_output - self.run() - - def insert(self, obj, direct_insertion = False): - if not direct_insertion: - if type(obj) == Blob: - self._tweak_blob(obj) - elif type(obj) == Commit: - aux_info = {'orig_parents': obj.parents, - 'had_file_changes': bool(obj.file_changes)} - self._tweak_commit(obj, aux_info) - elif type(obj) == Reset: - self._tweak_reset(obj) - elif type(obj) == Tag: - self._tweak_tag(obj) - self._insert_into_stream(obj) - - def _insert_into_stream(self, obj): - if not obj.dumped: - if self._parser: - self._parser.insert(obj) - else: - obj.dump(self._output) - - def get_exported_and_imported_refs(self): - return self._parser.get_exported_and_imported_refs() - - def run(self): - start = time.time() - if not self._input and not self._output: - self._run_sanity_checks() - if not self._args.dry_run and not self._args.partial: - self._migrate_origin_to_heads() - self._setup_input(use_done_feature = True) - self._setup_output() - assert self._sanity_checks_handled - - if self._input: - # Create and run the filter - self._repo_working_dir = self._args.source or b'.' - self._parser = FastExportParser(blob_callback = self._tweak_blob, - commit_callback = self._tweak_commit, - tag_callback = self._tweak_tag, - reset_callback = self._tweak_reset, - done_callback = self._final_commands) - self._parser.run(self._input, self._output) - if not self._finalize_handled: - self._final_commands() - - # Make sure fast-export completed successfully - if not self._args.stdin and self._fep.wait(): - raise SystemExit(_("Error: fast-export failed; see above.")) # pragma: no cover - self._input.close() - - # If we're not the manager of self._output, we should avoid post-run cleanup - if not self._managed_output: - return - - # Close the output and ensure fast-import successfully completes - self._output.close() - if not self._args.dry_run and self._fip.wait(): - raise SystemExit(_("Error: fast-import failed; see above.")) # pragma: no cover - - # With fast-export and fast-import complete, update state if requested - if self._args.state_branch: - self._save_marks_files() - - # Notify user how long it took, before doing a gc and such - msg = "New history written in {:.2f} seconds..." - if self._args.repack: - msg = "New history written in {:.2f} seconds; now repacking/cleaning..." - print(msg.format(time.time()-start)) - - # Exit early, if requested - if self._args.dry_run: - print(_("NOTE: Not running fast-import or cleaning up; --dry-run passed.")) - if self._fe_orig: - print(_(" Requested filtering can be seen by comparing:")) - print(" " + decode(self._fe_orig)) - else: - print(_(" Requested filtering can be seen at:")) - print(" " + decode(self._fe_filt)) - return - - target_working_dir = self._args.target or b'.' - if self._input: - self._ref_update(target_working_dir) - - # Write out data about run - self._record_metadata(self.results_tmp_dir(), self._orig_refs) - - # Final cleanup: - # If we need a repack, then nuke the reflogs and repack. - # If we need a reset, do a reset --hard - reset = not GitUtils.is_repository_bare(target_working_dir) - RepoFilter.cleanup(target_working_dir, self._args.repack, reset, - run_quietly=self._args.quiet, - show_debuginfo=self._args.debug) - - # Let user know how long it took - print(_("Completely finished after {:.2f} seconds.") - .format(time.time()-start)) - -def main(): - setup_gettext() - args = FilteringOptions.parse_args(sys.argv[1:]) - if args.analyze: - RepoAnalyze.run(args) - else: - filter = RepoFilter(args) - filter.run() - -if __name__ == '__main__': - main()