Edit File: svn-vendor
#!/usr/bin/python3 # vim: set sw=4 expandtab : # ==================================================================== # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # ==================================================================== # ############################################################################## # svn-vendor.py # # Overview # -------- # Replacement for svn_load_dirs.pl (included as a 'contributed utility' in # Subversion sources). Main difference is some heuristics in detection of # the renames. Note that this script does not attempt to automate remote # SVN operations (check-out, check-in and tagging), so it is possible to # review the state of sources that are about to be checked in. Another # difference is an ability to save the detected renames, review/re-apply # them. # # This script requires Python 3.3.x or higher. Sorry, I was too lazy # to write shell quoting routines that are already available in recent # Python versions. # # Using this script # ----------------- # First, it is necessary to check out the working copy from the URL that # will host the imported sources. E.g., if the versions of FOO are being # imported into svn://example.com/vendor/FOO/current: # # svn co svn://example.com/vendor/FOO/current wc # # Then, unpack the sources of the version to be imported: # # tar xzf foo-1.1.tar.gz # # Examples below assume the command above created a `foo-1.1' directory. # After that, there are three different modes of operation: # # 1. Fully automatic # # svn-vendor.py --auto wc foo-1.1 # svn st wc # svn ci wc # # In this mode, the script fully relies on its heuristics in detection of # renames. In many cases, it "just works". There can be spurious moves # detected in this mode, though. For example, consider a deleted header # that consists of 50 lines of GPL text, 1 line of copyright, and # 3 lines of declarations, and a similar unrelated header in the imported # sources. From the script's point of view, the files are nearly identical # (4 lines removed, 4 lines added, 50 lines unchanged). # # After the script completes, examine the working copy by doing 'svn diff' # and/or 'svn status', paying particular attention to renames. If all the # moves are detected correctly, check in the changes in the working copy. # # 2. Semi-automatic # # svn-vendor.py --detect moves-foo-1.1.txt wc foo-1.1 # vi moves-foo-1.1.txt # svn-vendor.py --apply moves-foo-1.1.txt wc foo-1.1 # svn ci wc # # If the fully automatic mode mis-detected some spurious moves, or did not # detect some renames you want to be performed, it is still possible to # leverage what the script has detected automatically. First command above # does the automatic detection, just as it does in fully automatic mode, # but stops short of performing any modification of the working copy. # The list of detected copies and renames is saved into a text file, # `moves-foo-1.1.txt'. # # That file can be inspected after the script finishes. Spurious moves can # be deleted from the file, and new copies/renames can be added. Then the # changes can be applied to the working copy. # # 3. Manual # # svn-vendor.py wc foo-1.1 # (svn-vendor) detect # (svn-vendor) move x.c y.c # (svn-vendor) move include/1.h include/2.h # (svn-vendor) copy include/3.h include/3-copy.h # (svn-vendor) lsprep # (svn-vendor) save /tmp/renames-to-be-applied.txt # (svn-vendor) apply # # If the automatic detection does not help, it is possible to do the renames # manually (similarly to svn_load_dirs.pl). Use the 'help' command to get # the list of supported commands and their description. Feel free to play # around - since the script does not perform any remote SVN operation, # there is no chance to commit the changes accidentally. # # Notes # ----- # I. The time for rename detection O(Fs*Fd) + O(Ds*Dd), where Fs is # the number of files removed from current directory, Fd is number of files # added in imported sources, and Ds/Dd is the same for directories. That is, # the running time may become an issue if the numbers of added/removed files # go into a few thousands (e.g. if updating Linux kernel 2.6.35 to 3.10). # As a workaround, import interim releases first so that the number of # renames remains sane at each step. That makes reviewing the renames # performed by the script much easier. # # Enjoy! # ############################################################################## import argparse import cmd import difflib import filecmp import os import readline import shlex import shutil import subprocess import sys def name_similarity(n1, n2): ''' Function to be used as a key for sorting dirs/files by name matching ''' sm = difflib.SequenceMatcher(a=n1, b=n2) return 1.0 - sm.ratio() def filename_sort_key(s): ''' Function to sort filenames so that parent directory is always followed by its children. Without it, [ "/a", "/a-b", "/a/b", "/a-b/c" ] would not be sorted correctly. ''' return s.replace('/', '\001') def descendant_or_self(path, ancestor): ''' Check if path is somewhere in hierarchy under ancestor. ''' return path == ancestor or path.startswith(ancestor + os.sep) def path_rebase(path, old_base, new_base): ''' Return a path name that has the same relative path to new_base as path had to old_base. Assumes path is a descendant of old_base. ''' if path == old_base: return new_base return os.path.normpath(os.path.join(new_base, os.path.relpath(path, old_base))) def for_all_parents(path, func): ''' Invoke func for each parent path. ''' d = os.path.dirname(path) while d != "": func(d) d = os.path.dirname(d) class InvalidUsageException(Exception): ''' Raised if command line arguments are invalid ''' def __init__(self, cmd, msg): Exception.__init__(self, msg) self.cmd = cmd class NotImplementedException(Exception): ''' Raised if some code path is not implemented ''' pass # Indexes into FSO.state S_WC = 0 S_IM = 1 class FSO(object): ''' File system object (file/dir either in imported dir or in WC) ''' def __init__(self): self.wc_path = None self.state = [ "-", "-" ] # '-': absent, 'F': file, 'D': dir, 'L': symlink def status(self): return "[%s%s]" % (self.state[S_WC], self.state[S_IM]) def orig_reference(self, curpath): if self.wc_path and self.wc_path != curpath: return " (original: %s)" % shlex.quote(self.wc_path) return "" class FSOCollection(dict): ''' Collection of FSOs ''' def print(self): print(" / Status in working copy (-:absent, F:file, D:dir, L:link)") print(" |/ Status in imported sources (-:absent, F:file, D:dir, L:link)") for k in sorted(self.keys(), key=filename_sort_key): e = self[k] print("%s %s%s" % (e.status(), shlex.quote(k), e.orig_reference(k))) def get(self, path): 'Get existing FSO or create a new one' if path in self: return self[path] e = FSO() self[path] = e return e def add(self, path, where, kind): 'Adding entries during initial scan' path = os.path.normpath(path) e = self.get(path) e.state[where] = kind if where == S_WC: e.wc_path = path def wc_copy(self, src, dst): 'Handle move in a working copy' keys = list(self.keys()) for k in keys: if descendant_or_self(k, src): esrc = self[k] if esrc.state[S_WC] == "-": continue kn = path_rebase(k, src, dst) edst = self.get(kn) if edst.state[S_WC] != "-": # Copying into existing destination. # Caller should've checked this. raise NotImplementedException edst.wc_path = esrc.wc_path edst.state[S_WC] = esrc.state[S_WC] def wc_remove(self, path): 'Handle removal in a working copy' keys = list(self.keys()) for k in keys: if descendant_or_self(k, path): self[k].state[S_WC] = "-" class ConfigOpt(object): 'Helper class - single option (string)' def __init__(self, value, helpmsg): self.value = value self.helpmsg = helpmsg def set(self, new_value): self.value = new_value def __str__(self): return "<none>" if self.value is None else "`%s'" % self.value class ConfigOptInt(ConfigOpt): 'Helper class - single option (integer)' def set(self, new_value): try: self.value = int(new_value) except ValueError: raise InvalidUsageException(None, "Value must be integer") def __str__(self): return "%d" % self.value class Config(dict): ''' Store configuration options. ''' def add_option(self, name, cfgopt): self[name] = cfgopt def set(self, name, value): if name not in self: raise InvalidUsageException(None, "Unknown config variable '%s'" % name) self[name].set(value) def get(self, name): if name not in self: raise NotImplementedException() return self[name].value def print(self): for k in sorted(self): o = self[k] for s in o.helpmsg.split('\n'): print("# %s" % s) print("%-20s: %s" % (k, str(o))) print("") class SvnVndImport(cmd.Cmd): ''' Main driving class. ''' intro = "Welcome to SVN vendor import helper. " + \ "Type help or ? to list commands.\n" prompt = "(svn-vendor) " prepare_ops = [] def __init__(self, wcdir, importdir, svninfo): cmd.Cmd.__init__(self) self.wcdir = wcdir self.importdir = importdir self.svninfo = svninfo self.config = Config() self.config.add_option('symlink-handling', ConfigOpt("as-is", "How symbolic links are handled;\n" + " 'dereference' treats as normal files/dirs (and " + "ignores dangling links);\n" + " 'as-is' imports as symlinks")) self.config.add_option('exec-permission', ConfigOpt("preserve", "How 'executable' permission bits " + "are handled;\n" + " 'preserve' sets svn:executable property as in " + "imported sources;\n" + " 'clear' removes svn:executable on all new files " + "(but keeps it intact on existing files).")) self.config.add_option('save-diff-copied', ConfigOpt(None, "Save 'svn diff' output on the " + "moved/copied files and directories to this " + "file as part of 'apply'")) self.config.add_option('dir-similarity', ConfigOptInt(600, "Similarity between dirs to assume " + "a copy/move [0..1000]")) self.config.add_option('file-similarity', ConfigOptInt(600, "Similarity between files to assume a " + "copy/move [0..1000]")) self.config.add_option('file-min-lines', ConfigOptInt(10, "Minimal number of lines in a file for " + "meaningful comparison")) self.config.add_option('verbose', ConfigOptInt(3, "Verbosity of the output [0..5]")) try: self.termwidth = os.get_terminal_size()[0] except OSError: # Not running in a terminal - probably redirected to file self.termwidth = 150 # arbitrary number def info(self, level, msg): 'Print message with specified verbosity' if level <= self.config.get('verbose'): print(msg, flush=True) def scan(self): self.items = FSOCollection() self.info(1, "Scanning working copy directory...") self.get_lists(self.wcdir, S_WC, False) self.info(1, "Scanning imported directory...") self.get_lists(self.importdir, S_IM, self.config.get('symlink-handling') == "dereference") def get_lists(self, top, where, deref): for d, dn, fn in os.walk(top, followlinks=deref): dr = os.path.relpath(d, top) # If under .svn directory at the top (SVN 1.7+) or has .svn # in the path (older SVN), ignore if descendant_or_self(dr, '.svn') or \ os.path.basename(dr) == '.svn' or \ (os.sep + '.svn' + os.sep) in dr: continue if dr != '.': self.items.add(dr, where, "D") dnn = [] # List where we'll descend for f in fn + dn: fr = os.path.normpath(os.path.join(dr, f)) frp = os.path.join(d, f) if os.path.islink(frp): if deref: # Dereferencing: # - check for dangling/absolute/out-of-tree symlinks and abort rl = os.readlink(frp) if not os.path.exists(frp): self.info(1, "WARN: Ignoring dangling symlink %s -> %s" % (fr, rl)) continue if os.path.isabs(rl): self.info(1, "WARN: Ignoring absolute symlink %s -> %s" % (fr, rl)) continue tgt = os.path.normpath(os.path.join(dr, rl)) if tgt == ".." or tgt.startswith(".." + os.sep): self.info(1, "WARN: Ignoring out-of-wc symlink %s -> %s" % (fr, rl)) continue else: # Importing symlinks as-is, no need to check. self.items.add(fr, where, "L") continue # If we get here, treat symlinks to files as regular files, and add directories # to the list of traversed subdirs if os.path.isfile(frp): self.items.add(fr, where, "F") if os.path.isdir(frp): dnn.append(f) dn[:] = dnn def onecmd(self, str): 'Override for checking number of arguments' try: return cmd.Cmd.onecmd(self, str) except InvalidUsageException as e: if e.cmd is not None: print("!!! Invalid usage of `%s' command: %s" % (e.cmd, e)) print("") self.onecmd("help " + e.cmd) else: print("!!! %s" % e) def parse_args(self, line, nargs, cmd): 'Parse arguments for a command' args = shlex.split(line) if len(args) != nargs: raise InvalidUsageException(cmd, "expect %d arguments" % nargs) return args def run_svn(self, args_fixed, args_split=[]): 'Run SVN command(s), potentially splitting long argument lists' rv = True pos = 0 atatime = 100 output = "" # svn treats '@' specially (peg revision); if there's such character in a # file name - append an empty peg revision args_fixed = list(map(lambda x: x + "@" if x.find("@") != -1 else x, args_fixed)) args_split = list(map(lambda x: x + "@" if x.find("@") != -1 else x, args_split)) while pos < len(args_split) or (pos == 0 and len(args_split) == 0): svnargs = ['svn'] + args_fixed + args_split[pos : pos + atatime] pos += atatime self.info(5, "Running: " + " ".join(map(shlex.quote, svnargs))) p = subprocess.Popen(args=svnargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.wcdir) so, se = p.communicate() if p.returncode != 0: print("`%s' exited with %d status:" % (" ".join(map(shlex.quote, svnargs)), p.returncode)) print(se.decode()) rv = False else: output += so.decode() return rv, output def copy_or_move(self, op, src, dst): 'Handle copy or move operation' if src not in self.items or self.items[src].state[S_WC] == "-": raise InvalidUsageException(None, "Nothing known about `%s'" % src) if dst in self.items and self.items[dst].state[S_WC] != "-": raise InvalidUsageException(None, "Destination path `%s' already exists" % dst) # Check that we're not creating dst under a file (not a dir) new_dirs = [] def check_parent(d): if d not in self.items or self.items[d].state[S_WC] == "-": new_dirs.append(d) elif self.items[d].state[S_WC] == "F": raise InvalidUsageException(None, "Destination path `%s' created under `%s' " + "which is a file" % (dst, d)) for_all_parents(dst, check_parent) # All ok, record new directories that may be created for d in new_dirs: self.items.get(d).state[S_WC] = "D" # Record the operation and update the FSO collection self.prepare_ops.append((op, src, dst)) self.items.wc_copy(src, dst) if op == "mv": self.items.wc_remove(src) def remove(self, path): if path not in self.items or self.items[path].state[S_WC] == "-": raise InvalidUsageException(None, "Nothing known about `%s'" % path) self.prepare_ops.append(("rm", path)) self.items.wc_remove(path) def similarity_file(self, src, dst, threshold, lst_removal): 'Compare two files, return similarity ratio on 0..1000 scale' if self.items[src].state[S_WC] != "F": return 0 # Source is in working copy fn1 = os.path.join(self.wcdir, self.items[src].wc_path) # Destination is in imported dir fn2 = os.path.join(self.importdir, dst) minlines = self.config.get('file-min-lines') try: f1 = open(fn1, 'r') l1 = f1.readlines() f1.close() if len(l1) < minlines: return 0 f2 = open(fn2, 'r') l2 = f2.readlines() f2.close() if len(l2) < minlines: return 0 sm = difflib.SequenceMatcher(a=l1, b=l2) return int(1000 * sm.quick_ratio()) except UnicodeDecodeError: # Oops, file seems to be binary. Fall back to comparing whole # file contents. if filecmp.cmp(fn1, fn2, shallow=False): return 1000 return 0 def _similarity_dir(self, src, dst, get_file_similarity, lst_removal): 'Iterate over FSOs, using callback to compare file entries' common = 0 total = 0 for xsrc in self.items: if xsrc.startswith(src + os.sep): esrc = self.items[xsrc] if esrc.state[S_WC] == "-": # Source not in WC - ignore for similarity calculation continue skip = False if lst_removal is not None: for i in lst_removal: if descendant_or_self(xsrc, i): skip = True if skip: # Moved to another place, do not consider in score continue total += 1000 xdst = path_rebase(xsrc, src, dst) if xdst not in self.items: # Destination not in imported sources - non-similar item continue edst = self.items[xdst] if edst.state[S_IM] == esrc.state[S_WC]: if esrc.state[S_WC] == "D": common += 1000 else: common += get_file_similarity(xsrc, xdst) if total == 0: # No files/subdirs in source directory - avoid copying empty dirs return 0 return 1000 * common / total def similarity_dir(self, src, dst, threshold, lst_removal): ''' Compare two dirs recursively, return similarity ratio on 0..1000 scale. ''' common = 0 total = 0 # Quickly estimate upper boundary by comparing file names. Only # concern ourselves with files in source directory. I.e., if # files were added after the move in the destination directory, # it's ok. If most of the files from the source directory were # removed, the directory is not considered similar - instead, # file move detection would move files one by one. upper = self._similarity_dir(src, dst, lambda s, d: 1000, lst_removal) if upper <= threshold: # Even the best estimate is worse than current cut-off return 0 # Okay, looks roughly similar. Now redo the above procedure, but also # compare the file content. return self._similarity_dir(src, dst, lambda s, d: self.similarity_file(s, d, 0, lst_removal), lst_removal) def similar(self, src, dst, threshold=0, lst_removal=None): 'Compare two FSOs, source in WC and destination in imported dir' if src not in self.items: print("Source `%s' not in the working copy" % src) return xsrc = self.items[src] if xsrc.state[S_WC] == "-": print("Source `%s' not in the working copy" % src) return if dst not in self.items: print("Destination `%s' not in imported sources" % dst) return xdst = self.items[dst] if xdst.state[S_IM] == "-": print("Destination `%s' not in imported sources" % dst) return if xsrc.state[S_WC] != xdst.state[S_IM]: # Different kinds - definitely not the same object return 0 if xsrc.state[S_WC] == "L" or xdst.state[S_IM] == "L": # Symlinks are not considered the same object (same target in # different dirs refers to different objects). return 0 if xsrc.state[S_WC] == "D": return self.similarity_dir(src, dst, threshold, lst_removal) else: return self.similarity_file(src, dst, threshold, lst_removal) def handle_op(self, op_tuple): 'Handle one SVN operation, recorded as a tuple' def x_mv(src, dst): self.info(2, " Move `%s' to `%s'" % (src, dst)) self.copy_or_move("mv", src, dst) def x_cp(src, dst): self.info(2, " Copy `%s' to `%s'" % (src, dst)) self.copy_or_move("cp", src, dst) def x_rm(path): self.info(2, " Remove `%s'" % path) self.remove(path) known_ops = { # key: (nargs, handler) 'cp' : (3, x_cp), 'mv' : (3, x_mv), 'rm' : (2, x_rm), } if len(op_tuple) == 0: raise InvalidUsageException op = op_tuple[0] if op not in known_ops: return False nargs, func = known_ops[op] if nargs != len(op_tuple): return False func(*op_tuple[1:]) return True def detect(self, thresholds): 'Helper for finding copy/move destinations' ilst = [] wlst = {} ilst_map = {} for p in self.items: e = self.items[p] if e.state[S_WC] != "-" and e.state[S_IM] == "-": wlst[p] = [] # wlst hash stores copy destinations elif e.state[S_WC] == "-" and e.state[S_IM] != "-": # ilst just lists destination paths as tuples with node kind ilst.append((e.state[S_IM], p)) iteration = 0 # Do not apply operations immediately - we'll need to post-process # them to account for files/dirs moved inside a moved parent dir. ops = [] to_be_removed = [] def get_renamed_name(path, rename_ops): ''' Check if path was renamed/removed in the recorded operations, return new name. ''' for op_tuple in rename_ops: # Since copies do not remove the source file, ignore them. # We push no 'rm' ops in this function if op_tuple[0] == "mv": src = op_tuple[1] dst = op_tuple[2] if descendant_or_self(path, src): path = path_rebase(path, src, dst) return path while len(wlst): iteration += 1 self.info(2, ("Iteration %d: Possible sources: %d, " + "possible destinations: %d") % (iteration, len(wlst), len(ilst))) ndst = len(ilst) for idx, (nk, dst) in enumerate(sorted(ilst, key=lambda s: filename_sort_key(s[1]))): class SkipDestFile(Exception): pass # Check if moved as a part of a parent directory. def check_moved_parent(xdst): if xdst in ilst_map: src = path_rebase(dst, xdst, ilst_map[xdst]) # Did it exist in copied directory? if src in self.items and \ self.items[src].state[S_WC] == nk: sim = self.similar(src, dst, thresholds[nk], to_be_removed) if sim > thresholds[nk]: self.info(2, (" [%04d/%04d] Skipping `%s' " + "(copied as part of `%s')") % (idx, ndst, dst, xdst)) raise SkipDestFile # Copied, not similar - search for other sources raise StopIteration try: for_all_parents(dst, check_moved_parent) except SkipDestFile: continue except StopIteration: pass self.info(2, (" [%04d/%04d] Looking for possible source " + "for `%s'") % (idx, ndst, dst)) bestsrc = None # Won't even consider those lower than threshold bestsim = thresholds[nk] for src in sorted(wlst.keys(), key=lambda x: name_similarity(x, dst)): sim = self.similar(src, dst, bestsim, to_be_removed) if sim > bestsim: self.info(3, " [similarity %4d] %s" % (sim, src)) bestsim = sim bestsrc = src if bestsim == 1000: # No chance we're finding anything better break if bestsrc is not None: wlst[bestsrc].append(dst) ilst_map[dst] = bestsrc # Discovered all copies/moves, now record them. new_wlst = {} for src in sorted(wlst.keys(), key=filename_sort_key): dlist = wlst[src] if len(dlist) == 0: continue if len(dlist) == 1: ops.append(("mv", src, dlist[0])) to_be_removed.append(src) else: # We don't remove the source here, it will be done when # the changes are applied (it will remove all the WC files # not found in imported sources). Avoiding removal here # simplifies operation sorting below, since we would not # be concerned with source file/dir disappearing before # it is copied to its destination. to_be_removed.append(src) for d in dlist: ops.append(("cp", src, d)) # If we copied something - recheck parent source directories. # Since some source file/dir was scheduled to be removed, # this may have increased the similarity to some destination. def recheck_parent(x): if x in wlst and len(wlst) == 0: new_wlst[x] = [] for_all_parents(src, recheck_parent) # At this point, if we're going to have the next iteration, we # are only concerned about directories (by the way new_wlst is # created above). So, filter out all files from ilst as well. wlst = new_wlst ilst = list(filter(lambda t: t[0] == 'D', ilst)) # Finished collecting the operations - now can post-process and # apply them. First, sort copies/moves by destination (so that # parent directories are created before files/subdirs are # copied/renamed inside) ops = sorted(ops, key=lambda op: filename_sort_key(op[2])) for i, op_tuple in enumerate(ops): # For each operation, go over its precedents to see if the source # has been renamed. If it is, find out new name. op = op_tuple[0] src = get_renamed_name(op_tuple[1], reversed(ops[:i])) if src != op_tuple[2]: # Unless it became the same file after renames try: # Try to remove the destination, if it existed self.remove(op_tuple[2]) except InvalidUsageException: # Okay, it didn't exist pass self.handle_op((op, src, op_tuple[2])) def do_detect(self, arg): ''' detect : auto-detect possible moves (where source/destination name is unique). If not all moves are applicable, save move list, edit and load. ''' self.parse_args(arg, 0, "detect") # Configurable for file/dirs; symlinks are never similar. self.detect({ "D": self.config.get('dir-similarity'), "F": self.config.get('file-similarity'), "L": 1001 }) def do_apply(self, arg): ''' apply : Perform copies/renames; then copy imported sources into the working copy. Modifies working copy. Exits after completion. ''' self.info(1, "Copying imported sources into working copy...") # Perform the recorded copies/moves/removals self.info(2, " Preparatory operations (copies/renames/removals)") to_be_diffed = [] for o in self.prepare_ops: op = o[0] if op == "mv": self.run_svn(["mv", "--parents", o[1], o[2]]) to_be_diffed.append(o[2]) elif op == "cp": self.run_svn(["cp", "--parents", o[1], o[2]]) to_be_diffed.append(o[2]) elif op == "rm": # --force, as the removed path is likely created as a result # of previous copy/rename self.run_svn(["rm", "--force", o[1]]) dirs_added = [] dirs_removed = [] files_added = [] files_removed = [] files_set_exec = [] files_clear_exec = [] self.info(2, " Creating dirs and copying files...") def copyfile_helper(i, nk_wc): '''Helper: copy a file and optionally, transfer permissions.''' f = os.path.join(self.importdir, i) t = os.path.join(self.wcdir, i) shutil.copyfile(f, t) # If exec-permission is 'clear', we don't need to do anything: # shutil.copyfile will create the file as non-executable. if self.config.get('exec-permission') == 'preserve': # If the file is new, just copying the mode is enough: # svn will set the svn:executable upon adding it. if nk_wc == "F": # Existing file, check what the setting shall be if os.access(f, os.X_OK) and not os.access(t, os.X_OK): files_set_exec.append(i) elif not os.access(f, os.X_OK) and os.access(t, os.X_OK): files_clear_exec.append(i) shutil.copymode(f, t) for i in sorted(self.items.keys()): e = self.items[i] nk_wc = e.state[S_WC] nk_im = e.state[S_IM] flg = None if nk_wc == "-": # Absent in working copy if nk_im == "D": # Directory added os.mkdir(os.path.join(self.wcdir, i)) dirs_added.append(i) flg = "(added dir)" elif nk_im == "F": # New file added copyfile_helper(i, nk_wc); files_added.append(i) flg = "(added file)" elif nk_im == "L": tim = os.readlink(os.path.join(self.importdir, i)) os.symlink(tim, os.path.join(self.wcdir, i)) files_added.append(i) flg = "(added symlink)" else: # Not in imported sources, not in WC (moved # away/removed) - nothing to do pass elif nk_wc == "L": # Symbolic link in a working copy if nk_im == "L": # Symbolic link in both. If the same target, do nothing. Otherwise, # replace. twc = os.readlink(os.path.join(self.wcdir, i)) tim = os.readlink(os.path.join(self.importdir, i)) if tim != twc: self.run_svn(["rm", "--force", i]) os.symlink(tim, os.path.join(self.wcdir, i)) files_added.append(i) flg = "(replaced symlink)" elif nk_im == "D": # Was a symlink, now a directory. Replace. self.run_svn(["rm", "--force", i]) os.mkdir(os.path.join(self.wcdir, i)) dirs_added.append(i) flg = "(replaced symlink with dir)" elif nk_im == "F": # Symlink replaced with file. self.run_svn(["rm", "--force", i]) copyfile_helper(i, nk_wc); files_added.append(i) flg = "(replaced symlink with file)" else: # Was a symlink, removed files_removed.append(i) flg = "(removed symlink)" elif nk_wc == "F": # File in a working copy if nk_im == "D": # File replaced with a directory. See comment above. self.run_svn(["rm", "--force", i]) os.mkdir(os.path.join(self.wcdir, i)) dirs_added.append(i) flg = "(replaced file with dir)" elif nk_im == "F": # Was a file, is a file - just copy contents copyfile_helper(i, nk_wc); flg = "(copied)" elif nk_im == "L": # Was a file, now a symlink. Replace. self.run_svn(["rm", "--force", i]) tim = os.readlink(os.path.join(self.importdir, i)) os.symlink(tim, os.path.join(self.wcdir, i)) files_added.append(i) flg = "(replaced file with symlink)" else: # Was a file, removed files_removed.append(i) flg = "(removed file)" elif nk_wc == "D": # Directory in a working copy if nk_im == "D": # Was a directory, is a directory - nothing to do pass elif nk_im == "F": # Directory replaced with file. Need to remove dir # immediately, as bulk removals/additions assume new files # and dirs already in place. Also, removing a directory # removes all its descendants - mark them as removed. self.run_svn(["rm", "--force", i]) self.items.wc_remove(i) copyfile_helper(i, nk_wc); files_added.append(i) flg = "(replaced dir with file)" elif nk_im == "L": # Was a directory, now a symlink. Replace. self.run_svn(["rm", "--force", i]) self.items.wc_remove(i) tim = os.readlink(os.path.join(self.importdir, i)) os.symlink(tim, os.path.join(self.wcdir, i)) files_added.append(i) flg = "(replaced dir with symlink)" else: # Directory removed dirs_removed.append(i) flg = "(removed dir)" if flg is not None: self.info(4, " %s %s %s" % (e.status(), i, flg)) # Filter files/directories removed as a part of parent directory files_removed = list(filter(lambda x: os.path.dirname(x) not in dirs_removed, files_removed)) dirs_removed = list(filter(lambda x: os.path.dirname(x) not in dirs_removed, dirs_removed)) files_added = list(filter(lambda x: os.path.dirname(x) not in dirs_added, files_added)) dirs_added = list(filter(lambda x: os.path.dirname(x) not in dirs_added, dirs_added)) self.info(2, " Running SVN add/rm/propset/propdel commands"); if len(dirs_added): self.run_svn(["add"], dirs_added) if len(files_added): self.run_svn(["add"], files_added) if len(dirs_removed): self.run_svn(["rm"], dirs_removed) if len(files_removed): self.run_svn(["rm"], files_removed) if len(files_set_exec): self.run_svn(["propset", "svn:executable", "*"], files_set_exec) if len(files_clear_exec): self.run_svn(["propdel", "svn:executable"], files_clear_exec) # Save the diff for the copied/moved items diff_save = self.config.get('save-diff-copied') if diff_save is not None: self.info(2, " Saving 'svn diff' on copied files/dirs to `%s'" % diff_save) to_be_diffed = list(filter(lambda x: os.path.dirname(x) not in to_be_diffed, to_be_diffed)) if len(to_be_diffed): try: rv, out = self.run_svn(["diff"], to_be_diffed) except UnicodeDecodeError: # Some binary files not marked with appropriate MIME type, # or broken text files rv, out = (True, "WARNING: diff contained binary files\n") else: rv, out = (True, "") if rv: f = open(diff_save, "w") f.write(out) f.close() # Exiting, as the resulting working copy can no longer be used # for move analysis self.info(1, "Done. Exiting; please examine the working copy " + "and commit.") return True def do_similarity(self, arg): ''' similarity SRD DST : estimate whether SRC could be potential source for DST (0=no match, 1000=perfect match) ''' src, dst = self.parse_args(arg, 2, "similarity") sim = self.similar(src, dst) if sim is not None: print("Similarity between source `%s' and destination `%s': %4d" % (src, dst, sim)) def do_set(self, arg): ''' set : display current settings set CFG VAL : set a config variable ''' if arg.strip() == '': self.config.print() else: cfg, val = self.parse_args(arg, 2, "set") self.config.set(cfg, val) def do_move(self, arg): ''' move SRC DST : Perform a move from source to destination ''' src, dst = self.parse_args(arg, 2, "move") self.copy_or_move("mv", src, dst) def do_copy(self, arg): ''' copy SRC DST : Perform a copy from source to destination ''' src, dst = self.parse_args(arg, 2, "copy") self.copy_or_move("cp", src, dst) def do_remove(self, arg): ''' remove PATH : Remove a path ''' path = self.parse_args(arg, 1, "remove")[0] self.copy_or_move("rm", path) def do_lsprep(self, arg): ''' lsprep : List the currently recorded moves/copies/removals ''' self.parse_args(arg, 0, "lsprep") colsz = int((self.termwidth - 14) / 2) if len(self.prepare_ops): print("Currently recorded preparatory operations:") print("") print("%5s %s %-*s %-*s" % ("#", "Op", colsz, "Source", colsz, "Destination")) for id, o in enumerate(self.prepare_ops): if id % 10 == 0: print("%5s %s %*s %*s" % ("-"*5, "--", colsz, "-"*colsz, colsz, "-"*colsz)) if len(o) == 3: print("%5d %s %-*s %-*s" % (id, o[0], colsz, o[1], colsz, o[2])) else: print("%5d %s %-*s" % (id, o[0], colsz, o[1])) print("") else: print("No copies/moves/removals recorded") print("") def do_save(self, arg): ''' save FILENAME : Save current preparation operations to a file ''' fn = self.parse_args(arg, 1, "save")[0] f = open(fn, 'w') longestname = 0 for o in self.prepare_ops: if len(o[1]) > longestname: longestname = len(o[1]) if len(o) == 3 and len(o[2]) > longestname: longestname = len(o[2]) for o in self.prepare_ops: if len(o) == 2: f.write("svn %s %-*s\n" % (o[0], longestname, shlex.quote(o[1]))) else: f.write("svn %s %-*s %-*s\n" % (o[0], longestname, shlex.quote(o[1]), longestname, shlex.quote(o[2]))) pass f.close() def do_load(self, arg): ''' load FILENAME : Load/append preparation operations from a file ''' fn = self.parse_args(arg, 1, "load")[0] self.info(1, "Performing operations from `%s'" % fn) f = open(fn, 'r') for l in f.readlines(): if l[0] == '#': continue args = shlex.split(l) try: if len(args) < 2 or args[0] != 'svn': raise InvalidUsageException(None, "") self.handle_op(args[1:]) except InvalidUsageException as e: # Rethrow raise InvalidUsageException(None, "Invalid line in file: %s(%s)" % (l, e)) f.close() def do_svninfo(self, arg): ''' svninfo : Display SVN info on the working copy (debug) ''' self.parse_args(arg, 0, "svninfo") print(str(self.svninfo)) def do_printlst(self, arg): ''' printlst WHAT : Print list of files; WHAT is one of {dir,file} (debug) ''' self.parse_args(arg, 0, "printlst") self.items.print() def do_help(self, arg): ''' help [COMMAND] : Print the help message ''' cmd.Cmd.do_help(self, arg) def do_EOF(self, arg): ''' Quit the script ''' return True def do_quit(self, arg): ''' quit : Quit the script ''' return True if __name__ == '__main__': parser = argparse.ArgumentParser( description="Prepare a working copy for SVN vendor import.") parser.add_argument('wcdir', help="Path to working copy (destination of import)") parser.add_argument('importdir', help="Path to imported sources (source of import)") grp = parser.add_mutually_exclusive_group() grp.add_argument('--auto', action='store_true', help="Automatic mode: detect moves, apply them and copy sources") grp.add_argument('--detect', metavar='FILE', help="Semi-automatic mode: detect moves and save them to FILE") grp.add_argument('--apply', metavar='FILE', help="Semi-automatic mode: apply the moves from FILE " + "and copy the sources") parser.add_argument('--save', metavar='FILE', help="Automatic mode: save moves to FILE after detection, " + "then proceed to apply the changes") parser.add_argument('--config', metavar=('OPT','VALUE'), action='append', nargs=2, help="Set configuration option OPT to VALUE") args = parser.parse_args() p = subprocess.Popen(args=['svn', 'info', args.wcdir], stdout=subprocess.PIPE, stderr=subprocess.PIPE) so, se = p.communicate() if p.returncode != 0: print("%s: does not appear to be SVN working copy." % args.wcdir) print("`svn info' exited with status %d and returned:" % p.returncode) print("") print(se.decode()) sys.exit(1) imp = SvnVndImport(args.wcdir, args.importdir, so.decode()) if args.config: try: for o, v in args.config: imp.config.set(o, v) except InvalidUsageException as e: parser.error(e) imp.scan() if args.auto: imp.onecmd("detect") if args.save: imp.onecmd("save " + shlex.quote(args.save)) imp.onecmd("apply") elif args.detect: imp.onecmd("detect") imp.onecmd("save " + shlex.quote(args.detect)) elif args.apply: imp.onecmd("load " + shlex.quote(args.apply)) imp.onecmd("apply") else: imp.cmdloop()