3rd-party-merge: add basic version matchingHEADmastermainbrillo-m10-releasebrillo-m10-dev

The original design was that people could only install the latest version
of packages and that's it.  Requests are now coming in to support older
SLOTs (like getting python-2.7).  That means we have to make the merger
more intelligent and support extended atoms like "dev-lang/python:2.7".

BUG=b:26861037
TEST=merging with python:2.7 works now

Change-Id: I6085adf87a37a601e2ebace9894ac5e558f448a4

1 files changed, 136 insertions, 19 deletions

diff --git a/usr/bin/3rd-party-merge b/usr/bin/3rd-party-merge
index 9cbde58..c28a2eb 100755
--- a/usr/bin/3rd-party-merge
+++ b/usr/bin/3rd-party-merge
@@ -20,8 +20,8 @@ from __future__ import print_function
 
 import argparse
 import errno
-import glob
 import os
+import re
 import shutil
 import sys
 
@@ -44,39 +44,149 @@ def unlink(path):
             raise
 
 
-def load_contents(pkg, root, filter=None):
-    """Read the CONTENTS file for |pkg| from |root|."""
-    if not filter:
-        filter = lambda x: True
+class Atom(object):
+    """Hold details about a package atom (optionally in a vdb)."""
 
-    ret = []
+    _PREFIX_RE = re.compile(r'^([>=<~!]*)(.*)$')
+    _PN_RE = r'(?P<package>[A-Za-z0-9_]+[A-Za-z0-9+_-]*?)'
+    _PV_RE = (r'(?P<version>[0-9](\.[0-9]+)*(_(pre|p|beta|alpha|rc)[0-9]*)*)'
+              r'(-r(?P<revision>[0-9]+))?')
+    _PN_PV_RE = re.compile(r'^%s(-%s)?$' % (_PN_RE, _PV_RE))
 
-    contents = os.path.join(root, 'var', 'db', 'pkg', pkg + '-*', 'CONTENTS')
-    files = glob.glob(contents)
-    if not files:
-        raise Exception('No packages found matching %s' % pkg)
+    def __init__(self, atom, path=None):
+        self.atom = atom
+        self.path = path
 
-    with open(files[0]) as f:
-        for line in f.read().splitlines():
+        # Extract any version matching prefixes.
+        m = self._PREFIX_RE.match(atom)
+        self.prefix, atom = m.groups()
+
+        # Split off trailing slot -- must be before category for subslots.
+        if ':' in atom:
+            atom, self._slot = atom.split(':', 1)
+        else:
+            self._slot = None
+
+        # Split off leading category.
+        if '/' in atom:
+            self.category, atom = atom.split('/', 1)
+        else:
+            self.category = None
+
+        # Split apart the name & version.
+        m = self._PN_PV_RE.match(atom)
+        self.pn = m.group('package')
+        self.pv = m.group('version')
+        self.rev = m.group('revision')
+
+    def _load_vdb_entry(self, entry):
+        if not self.path:
+            return None
+        path = os.path.join(self.path, entry)
+        with open(path) as f:
+            return f.read().strip()
+
+    @property
+    def slot(self):
+        if self._slot is None and self.path:
+            self._slot = self._load_vdb_entry('SLOT')
+        return self._slot
+
+    def load_contents(self, path_filter=None):
+        """Read the CONTENTS file for |pkg| from |root|."""
+        if not path_filter:
+            path_filter = lambda x: True
+
+        ret = []
+
+        for line in self._load_vdb_entry('CONTENTS').splitlines():
             line = line.strip()
             if not line:
                 continue
             typ, data = line.split(' ', 1)
             if typ == 'obj':
                 path, _hash, _mtime = data.rsplit(' ', 2)
-                if filter(path):
+                if path_filter(path):
                     ret.append(('obj', path))
             elif typ == 'sym':
                 source, target = data.split(' -> ', 1)
                 target, _mtime = target.rsplit(' ', 1)
-                if filter(source):
+                if path_filter(source):
                     ret.append(('sym', source, target))
             elif typ == 'dir':
                 pass
             else:
                 raise Exception('Unhandled entry: %s' % line)
 
-    return ret
+        return ret
+
+    def match(self, atom):
+        """See if |atom| matches us."""
+        # TODO: Support the range matching like >=.
+        if self.prefix not in (None, '', '=', '~'):
+            raise ValueError('Only exact matches supported (not %s)'
+                             % (self.prefix))
+
+        # Package names must always exist & match.
+        if atom.pn != self.pn:
+            return False
+
+        # Only require the category to match if both atoms specify it.
+        if len(set((atom.category, self.category, None))) == 3:
+            return False
+
+        # Only require the slot to match if both atoms specify it.
+        if self.slot and len(set((atom.slot, self.slot, None))) == 3:
+            return False
+
+        # Only require the version to match if both atoms specify it.
+        if len(set((atom.pv, self.pv, None))) == 3:
+            return False
+
+        # Only require the revision to match if both atoms specify it.
+        if (self.prefix is not '~' and
+            len(set((atom.rev, self.rev, None)))) == 3:
+            return False
+
+        # If we're still here, there's nothing left to say :).
+        return True
+
+    def __str__(self):
+        return ('Atom(%s/%s-%s-r%s:%s)' %
+                (self.category, self.pn, self.pv, self.rev, self.slot))
+
+
+class Vdb(object):
+    """Hold details about a vdb (a database of installed packages)."""
+
+    # https://projects.gentoo.org/pms/6/pms.html#x1-180003.1.1
+    CATEGORY_RE = re.compile(r'^[A-Za-z0-9_]+[A-Za-z0-9+_.-]*$')
+
+    # https://projects.gentoo.org/pms/6/pms.html#x1-200003.1.2
+    PACKAGE_RE = re.compile(r'^[A-Za-z0-9_]+[A-Za-z0-9+_-]*-[0-9]+')
+
+    def __init__(self, root):
+        self.pkgs = []
+
+        path = os.path.join(root, 'var', 'db', 'pkg')
+        for cat in os.listdir(path):
+            catdir = os.path.join(path, cat)
+            if not os.path.isdir(catdir) or not self.CATEGORY_RE.match(cat):
+                continue
+
+            for pkg in os.listdir(catdir):
+                pkgdir = os.path.join(catdir, pkg)
+                if (os.path.isdir(pkgdir) and
+                    os.path.isfile(os.path.join(pkgdir, 'CONTENTS')) and
+                    self.PACKAGE_RE.match(pkg)):
+                    self.pkgs.append(Atom('%s/%s' % (cat, pkg), pkgdir))
+
+    def find_pkg(self, match_pkg):
+        """Try to find the best match for |match_pkg|."""
+        match_atom = Atom(match_pkg)
+        for vdb_pkg in self.pkgs:
+            if match_atom.match(vdb_pkg):
+                return vdb_pkg
 
 
 def ignore_files(path):
@@ -102,16 +212,20 @@ def ignore_files(path):
     return True
 
 
-def merge(pkg, input_root, output_root, make_root, verbose=0):
+def merge(vdb, pkg, input_root, output_root, make_root, verbose=0):
     """Merge |pkg| from |input_root| to |output_root|."""
     print('Merging %s ' % pkg, end='')
     if verbose:
-        print('from %s to %s' % (input_root, output_root))
+        print('from %s to %s ' % (input_root, output_root))
 
     # TODO: Grab a lock for |pkg|.
 
     # First get the file listing to merge.
-    contents = load_contents(pkg, input_root, filter=ignore_files)
+    vdb_pkg = vdb.find_pkg(pkg)
+    if not vdb_pkg:
+        raise Exception('No packages found matching %s ' % pkg)
+    print('matched %s ' % vdb_pkg, end='')
+    contents = vdb_pkg.load_contents(path_filter=ignore_files)
 
     # Now actually merge them.
     for entry in contents:
@@ -123,6 +237,8 @@ def merge(pkg, input_root, output_root, make_root, verbose=0):
             else:
                 print('.', end='')
             makedirs(os.path.dirname(output))
+            if os.path.exists(output):
+                os.chmod(output, 0o666)
             shutil.copy2(os.path.join(input_root, path), output)
         elif entry[0] == 'sym':
             path = entry[1].lstrip('/')
@@ -167,8 +283,9 @@ def main(argv):
     parser = get_parser()
     opts = parser.parse_args(argv)
 
+    vdb = Vdb(opts.input_root)
     for pkg in opts.packages:
-        merge(pkg, opts.input_root, opts.output_root, opts.make_root,
+        merge(vdb, pkg, opts.input_root, opts.output_root, opts.make_root,
               verbose=opts.verbose)