#!/usr/bin/python ''' cyg-apt - Keep Cygwin or Mingw root up to date (c) 2002--2006 Jan Nieuwenhuizen License: GNU GPL ''' ''' cyg-apt is intended to keep a Cygwin cross compilation tree up to date, but can also be used to download packages for off line use. For example: download LilyPond, with all dependencies wget -P/tmp/lily http://cygwin.com/setup.exe wget -P/tmp/lily http://lilypond.org/cygwin/cyg-apt chmod +x /tmp/lily/cyg-apt # :-) /tmp/lily/cyg-apt --root=/tmp/lily --cache=/tmp/lily setup /tmp/lily/cyg-apt --download install lilypond zip -r /tmp/lily.zip /tmp/lily ''' ''' TODO * Get packagers to fix postinstall scripts - libncursses-devel prefix="$(cd $(dirname $0)/../../usr && pwd)" - libpng12-devel prefix="$(cd $(dirname $0)/../../usr && pwd)" ''' import __main__ import getopt import os import re import shutil import string import sys try: import urllib except: # Work around Cygwin-Python dll brokenness def ugh_quote (x, s, safe = '/'): res = list(s) for i in range(len(res)): c = res[i] if c not in safe: res[i] = '%%%02X' % ord (c) return ''.join (res) class ugh_urllib: pass urllib = ugh_urllib () urllib.quote = ugh_quote pass try: fake_pipe = 0 date = os.popen ('date').read () except: # Work around Cygwin-Python pipe brokenness ##import tempfile def fake_pipe (command, mode = 'r'): if mode == 'w': raise 'ugh' ##h, name = tempfile.mkstemp ('pipe', basename, '/tmp')x name = ('/tmp/%s.%d' % ('cyg-apt', os.getpid ())) os.system (command + ' > ' + name) return open (name) os.popen = fake_pipe pass basename = os.path.basename (sys.argv[0]) target = 'cygwin' if basename[:3] == 'min': target = 'mingw' HOME = os.environ['HOME'] CWD = os.getcwd () MKNETREL = CWD NETREL = '%(MKNETREL)s/%(target)s' % vars () ROOT = '%(NETREL)s/root' % vars () mknetrel_rc = HOME + '/.mknetrel' home_cyg_apt_rc = HOME + '/.' + basename cwd_cyg_apt_rc = CWD + '/.' + basename cygwin_p = os.uname ()[0][:6] == 'CYGWIN' if cygwin_p: ROOT = '/.' if target == 'mingw': ##ROOT = '/cygdrive/c/mingw' PFD_key = '/machine/Software/Microsoft/Windows/CurrentVersion/ProgramFilesDir' pfd = os.popen ("regtool get %(PFD_key)s | cygpath -uf-" % vars ()).read ()[:-1] ROOT = pfd + '/LilyPond' if not os.path.exists (ROOT): os.makedirs (ROOT) config = ROOT + '/etc/setup' setup_ini = config + '/setup.ini' EXTRA = MKNETREL + '/extra' PATCH = MKNETREL + '/patch' SRC = NETREL + '/src' os.environ['PATH'] = MKNETREL + '/bin:' + os.environ['PATH'] INSTALL = 'install' installed_db_magic = 'INSTALLED.DB 2\n' distname = 'curr' local_mirror = 'file://' + NETREL + '/uploads/..' official_mirror = 'http://mirrors.rcn.net/pub/sourceware/cygwin' official_mirror = 'http://gnu.kookel.org/ftp/cygwin' if target == 'mingw': official_mirror = 'http://lilypond.org/mingw' mirror = local_mirror #if not os.path.exists (local_mirror[7:]): if not os.path.exists (local_mirror[7:] + "/setup.ini"): mirror = official_mirror cache = ROOT + '/var/cache/setup' downloads = cache + '/' + urllib.quote (mirror, '').lower () rc_options = ['ROOT', 'MKNETREL', 'NETREL', 'mirror', 'cache', 'setup_ini', 'distname'] h = 0 if os.path.exists (cwd_cyg_apt_rc): h = open (cwd_cyg_apt_rc) elif os.path.exists (home_cyg_apt_rc): h = open (home_cyg_apt_rc) if h: for i in h.readlines (): k, v = i.split ('=', 2) if k in rc_options: __main__.__dict__[k] = eval (v) h.close () config = ROOT + '/etc/setup' downloads = cache + '/' + urllib.quote (mirror, '').lower () installed_db = config + '/installed.db' def usage (): sys.stdout.write ('''%s [OPTION]... COMMAND [PACKAGE]... Commands: ''' % basename) d = __main__.__dict__ commands = filter (lambda x: type (d[x]) == type (usage) and d[x].__doc__, d) sys.stdout.writelines (map (lambda x: " %s - %s\n" % (x, d[x].__doc__), psort (commands))) sys.stdout.write (r''' Options: -c,--cache=DIR download cache [%(cache)s] -d,--download download only -h,--help show brief usage -i,--ini=FILE use setup.ini [%(setup_ini)s] -m,--mirror=URL use mirror [%(mirror)s] -n,--netrel=DIR set netrel dir [%(NETREL)s] -r,--root=DIR set %(target)s root [%(ROOT)s] -t,--dist=NAME set dist name (curr, test, prev) [%(distname)s] -x,--no-deps ignore dependencies ''' % d) (options, files) = getopt.getopt (sys.argv[1:], 'c:dhi:m:r:t:x', ('cache=', 'download', 'help', 'mirror=', 'root=', 'ini=', 'dist=', 'no-deps')) command = 'help' if len (files) > 0: command = files[0] packagename = 0 if len (files) > 1: packagename = files[1] nodeps_p = 0 download_p = 0 for i in options: o = i[0] a = i[1] if 0: pass elif o == '--cache' or o == '-c': cache = a downloads = cache + '/' + urllib.quote (mirror, '').lower () elif o == '--download' or o == '-d': download_p = 1 elif o == '--help' or o == '-h': command = 'help' elif o == '--ini' or o == '-i': setup_ini = a elif o == '--mirror' or o == '-m': mirror = a downloads = cache + '/' + urllib.quote (mirror, '').lower () elif o == '--root' or o == '-r': ROOT = a config = ROOT + '/etc/setup' cache = ROOT + '/var/cache/setup' setup_ini = config + '/setup.ini' installed_db = config + '/installed.db' downloads = cache + '/' + urllib.quote (mirror, '').lower () elif o == '--dist' or o == '-t': distname = a elif o == '--no-deps' or o == '-x': nodeps_p = 1 home_based_p = ROOT == os.path.abspath (ROOT) if home_based_p: cyg_apt_rc = home_cyg_apt_rc else: cyg_apt_rc = cwd_cyg_apt_rc h = open (cyg_apt_rc, 'w') for i in rc_options: h.write ('%s="%s"\n' % (i, __main__.__dict__[i])) h.close () def version_to_string (t): def try_itoa (x): if type (x) == int: return "%d" % x return x return '%s-%s' % (string.join (map (try_itoa, t[:-1]), '.'), t[-1]) def string_to_version (s): s = re.sub ('([^0-9][^0-9]*)', ' \\1 ', s) s = re.sub ('[ _.-][ _.-]*', ' ', s) def try_atoi (x): if re.match ('^[0-9]*$', x): return string.atoi (x) return x return tuple (map (try_atoi, (string.split (s, ' ')))) def split_ball (p): m = re.match ('^(.*)-([0-9].*-[0-9]+)(.tar.bz2)?$', p) if not m: print 'split_ball: ' + p return (p[:2], (0, 0)) t = (m.group (1), string_to_version (m.group (2))) return t def join_ball (t): return t[0] + '-' + version_to_string (t[1]) ########################### def debug (s): s def uri_get (dir, uri): if uri[:7] == 'file://': return os.system ('cp -pv "%s" "%s"' % (uri[7:], dir)) else: return os.system ('cd "%s" && wget -c "%s"' % (dir, uri)) def help (): '''help COMMAND''' if len (files) < 2: usage () sys.exit () print __main__.__dict__[packagename].__doc__ dists = 0 distnames = ('curr', 'test', 'prev') def get_setup_ini (): global dists if dists: return dists = {'test': {}, 'curr': {}, 'prev' : {}} chunks = string.split (open (setup_ini).read (), '\n\n@ ') for i in chunks[1:]: lines = string.split (i, '\n') name = string.strip (lines[0]) debug ('package: ' + name) packages = dists['curr'] records = {'sdesc': name} j = 1 while j < len (lines) and string.strip (lines[j]): debug ('raw: ' + lines[j]) if lines[j][0] == '#': j = j + 1 continue elif lines[j][0] == '[': debug ('dist: ' + lines[j][1:5]) packages[name] = records.copy () packages = dists[lines[j][1:5]] j = j + 1 continue try: key, value = map (string.strip, string.split (lines[j], ': ', 1)) except: print lines[j] raise 'URG' if value[0] == '"' and value.find ('"', 1) == -1: while 1: j = j + 1 value += '\n' + lines[j] if lines[j].find ('"') != -1: break records[key] = value j = j + 1 packages[name] = records def get_url (): if not dists[distname].has_key (packagename) \ or not dists[distname][packagename].has_key (INSTALL): no_package () install = 0 for d in distnames: if dists[d].has_key (packagename) \ and dists[d][packagename].has_key (INSTALL): install = dists[d][packagename][INSTALL] sys.stderr.write ("warning: using [%s]\n" % d) break if not install: sys.stderr.write ("warning: %s no install\n" \ % packagename) return 0 else: install = dists[distname][packagename][INSTALL] file, size, md5 = string.split (install) return file, md5 def url (): '''print tarball url''' print get_url ()[0] def get_ball (): url, md5 = get_url () return '%s/%s' % (downloads, url) def ball (): '''print tarball name''' print get_ball () def do_download (): url, md5 = get_url () dir = '%s/%s' % (downloads, os.path.split (url)[0]) if not os.path.exists (get_ball ()) or not check_md5 (): if not os.path.exists (dir): os.makedirs (dir) # urllib status = uri_get (dir, '%s/%s' % (mirror, url)) # successful pipe close returns 'None' if not status: status = 0 signal = 0x0f & status ## exit_status = status >> 8 if status: raise 'urg' def download (): '''download package''' do_download () ball () md5 () print def no_package (s='error'): sys.stderr.write ("%s: %s not in [%s]\n" % (s, packagename, distname)) def get_requires (): dist = dists[distname] if not dists[distname].has_key (packagename): no_package ('error') #return [] sys.exit (1) if nodeps_p: return [packagename] reqs = {packagename:0} if INSTALL == 'source' \ and dist[packagename].has_key ('external-source'): reqs[dist[packagename]['external-source']] = 0 n = 0 while len (reqs) > n: n = len (reqs) for i in reqs.keys (): if not dist.has_key (i): sys.stderr.write ("error: %s not in [%s]\n" \ % (i, distname)) if i != packagename: del reqs[i] continue reqs[i] = '0' p = dist[i] if not p.has_key ('requires'): continue reqs.update (dict (map (lambda x: (x, 0), string.split (p['requires'])))) return reqs.keys () def requires (): '''print requires: for package''' print string.join (get_requires (), '\n') def buildrequires (): '''print buildrequires: for package''' global INSTALL INSTALL = 'source' print string.join (get_requires (), '\n') installed = 0 def get_installed (): global installed if installed: return installed installed = {0:{}} for i in open (installed_db).readlines ()[1:]: name, ball, status = string.split (i) installed[int (status)][name] = ball return installed def write_installed (): file = open (installed_db, 'w') file.write (installed_db_magic) file.writelines (map (lambda x: '%s %s 0\n' % (x, installed[0][x]), installed[0].keys ())) if file.close (): raise 'urg' def get_field (field, default=''): for d in (distname,) + distnames: if dists[d].has_key (packagename) \ and dists[d][packagename].has_key (field): return dists[d][packagename][field] return default def psort (lst): plist.sort (lst) return lst #urg plist = list def list (): '''installed packages''' global packagename for packagename in psort (installed[0].keys ()): ins = get_installed_version () new = 0 if dists[distname].has_key (packagename) \ and dists[distname][packagename].has_key (INSTALL): new = get_version () s = '%-20s%-15s' % (packagename, version_to_string (ins)) if new and new != ins: s += '(%s)' % version_to_string (new) print s def filelist (): '''installed files''' print string.join (get_filelist (), '\n') def update (): '''setup.ini''' if not os.path.exists (downloads): os.makedirs (downloads) os.system ('rm -f "%s/%s"' % (downloads, 'setup.ini')) uri_get (downloads, '%s/%s' % (mirror, 'setup.ini')) if os.path.exists (setup_ini): os.system ('cd %s && mv -f setup.ini setup.ini~' % config) os.system ('cp -pf "%s/setup.ini" "%s"' % (downloads, config)) def get_version (): if not dists[distname].has_key (packagename) \ or not dists[distname][packagename].has_key (INSTALL): no_package () return (0, 0) package = dists[distname][packagename] if not package.has_key ('ver'): file = string.split (package[INSTALL])[0] ball = os.path.split (file)[1] package['ver'] = split_ball (ball)[1] return package['ver'] def get_installed_version (): return split_ball (installed[0][packagename])[1] def version (): '''print installed version''' global distname, packagename if packagename: if not installed[0].has_key (packagename): distname = 'installed' no_package () sys.exit (1) print version_to_string (get_installed_version ()) else: for packagename in psort (installed[0].keys ()): if not installed[0].has_key (packagename): distname = 'installed' no_package () sys.exit (1) print '%-20s%-12s' % (packagename, version_to_string (get_installed_version ())) def get_new (): global packagename lst = [] for packagename in installed[0].keys (): new = get_version () ins = get_installed_version () if new > ins: debug (" %s > %s" % (new, ins)) lst.append (packagename) return lst def new (): '''list new (upgradable) packages in distribution''' #print string.join (get_new (), '\n') global packagename for packagename in psort (get_new ()): print '%-20s%-12s' % (packagename, version_to_string (get_version ())) def get_md5 (): url, md5 = get_url () pipe = os.popen ('md5sum "%s/%s"' % (downloads, url), 'r') actual_md5 = string.split (pipe.read ())[0] return actual_md5 def check_md5 (verbose=0): return get_url ()[1] == get_md5 () def md5 (): '''check md5 sum''' url, md5 = get_url () ball = os.path.basename (url) print '%s %s' % (md5, ball) actual_md5 = get_md5 () print '%s %s' % (actual_md5, ball) if actual_md5 != md5: raise 'URG' def search (): '''search package list''' global packagename regexp = packagename packages = [] keys = [] if distname in dists: keys = dists[distname].keys () else: for i in dists.keys (): for j in dists[i].keys (): if not j in keys: keys.append (j) for i in keys: packagename = i #if not regexp or re.search (regexp, i): if not regexp or re.search (regexp, i) \ or re.search (regexp, get_field ('sdesc')) \ or re.search (regexp, get_field ('ldesc')): if distname in dists: if dists[distname][i].has_key (INSTALL): packages.append (i) else: packages.append (i) for packagename in psort (packages): s = packagename d = get_field ('sdesc') if d: s += ' - %s' % d[1:-1] print s def show (): '''print information for package''' s = packagename d = get_field ('sdesc') if d: s += ' - %s' % d[1:-1] print s print print get_field ('ldesc') def get_missing (): reqs = get_requires () lst = [] for i in reqs: if not installed[0].has_key (i): lst.append (i) if lst and packagename not in lst: sys.stderr.write ('warning: missing packages: %s\n' % string.join (lst)) elif installed[0].has_key (packagename): ins = get_installed_version () new = get_version () if ins >= new: sys.stderr.write ('%s is already the newest version\n' % packagename) #lst.remove (packagename) elif packagename not in lst: lst.append (packagename) return lst def missing (): '''print missing dependencies''' print string.join (get_missing (), '\n') def run_script (file_name): sys.stderr.write ('running: %(file_name)s\n' % vars ()) os.system ('sh "%(file_name)s" && mv "%(file_name)s" "%(file_name)s.done"' % vars ()) def try_run_script (file_name): if os.path.isfile (file_name): if cygwin_p: run_script (file_name) else: sys.stderr.write ('warning: please see after: %(file_name)s' % vars ()) sys.stderr.write ('\n') def run_all (dir): if os.path.isdir (dir): #lst = filter (lambda x: x[-5:] != '.done', os.listdir (dir)) lst = filter (lambda x: x[-3:] == '.sh', os.listdir (dir)) for i in lst: try_run_script ('%s/%s' % (dir, i)) def do_install (): # find ball ball = get_ball () # untar capture list # tarfile pipe = os.popen ('tar -C "%s" -xjvf "%s"' % (ROOT, ball), 'r') lst = map (string.strip, pipe.readlines ()) if pipe.close (): raise 'urg' # write list write_filelist (lst) ## run_all (ROOT + '/etc/postinstall') #update installed[] installed[0][packagename] = os.path.basename (ball) # write installed.db write_installed () def get_filelist (): pipe = os.popen ('gzip -dc "%s/%s.lst.gz"' % (config, packagename), 'r') lst = map (string.strip, pipe.readlines ()) if pipe.close (): raise 'urg' return lst def write_filelist (lst): lst_name = '%s/%s.lst' % (config, packagename) if not fake_pipe: pipe = os.popen ('gzip -c > "%s.gz"' % lst_name, 'w') else: pipe = open (lst_name, 'w') for i in lst: pipe.write (i) pipe.write ('\n') if pipe.close (): raise 'urg' if fake_pipe: os.system ('gzip -f "%s"' % lst_name) os.system ('touch -r %s %s.gz' % (setup_ini, lst_name)) def do_uninstall (): try_run_script (ROOT + '/etc/preremove/%s.sh' % packagename) postremove = ROOT + '/etc/postremove/%s.sh' % packagename # get file list lst = get_filelist () # remove files for i in lst: file = os.path.join (ROOT, i) if not os.path.exists (file) and not os.path.islink (file): sys.stderr.write ('warning: %s no such file\n' % file) elif not os.path.isdir (file) and file != postremove: if os.remove (file): raise 'urg' try_run_script (postremove) if os.path.isfile (postremove): if os.remove (postremove): raise 'urg' # remove empty dirs? # cleanup write_filelist ([]) # update installed[] del (installed[0][packagename]) write_installed () def remove (): '''uninstall packages''' global packagename for packagename in files[1:]: if not installed[0].has_key (packagename): sys.stderr.write ('warning: %s not installed\n' % packagename) continue sys.stderr.write ('removing %s %s\n' \ % (packagename, version_to_string (get_installed_version ()))) do_uninstall () def install (): '''download and install packages with dependencies''' global packagename missing = {} for packagename in files[1:]: missing.update (dict (map (lambda x: (x, 0), get_missing ()))) if len (missing) > 1: sys.stderr.write ('to install: \n') sys.stderr.write (' %s' % string.join (missing.keys ())) sys.stderr.write ('\n') for packagename in missing.keys (): if not get_url (): del missing[packagename] for packagename in missing.keys (): download () if download_p: sys.exit (0) for packagename in missing.keys (): if installed[0].has_key (packagename): sys.stderr.write ('preparing to replace %s %s\n' \ % (packagename, version_to_string (get_installed_version ()))) do_uninstall () sys.stderr.write ('installing %s %s\n' \ % (packagename, version_to_string (get_version ()))) do_install () run_all (ROOT + '/etc/postinstall') def upgrade (): '''all installed packages''' files[1:] = get_new () install () def setup (): '''cygwin environment''' if not os.path.isdir (ROOT): sys.stderr.write ('error: %s no root dir\n' % ROOT) sys.exit (2) if not os.path.isdir (config): sys.stderr.write ('creating %s\n' % config) os.makedirs (config) if not os.path.isfile (installed_db): sys.stderr.write ('creating %s\n' % installed_db) global installed installed = {0:{}} write_installed () if not os.path.isfile (setup_ini): sys.stderr.write ('getting %s\n' % setup_ini) update () if not os.path.isfile (mknetrel_rc): mknetrel_root = os.path.abspath (MKNETREL) netrel_root = os.path.abspath (NETREL) cygwin_root = os.path.abspath (ROOT) cygcomment = '' mincomment = '#' if target == 'mingw': cygcomment = '#' mincomment = '' open (mknetrel_rc, 'w').write ('''# -*-shell-script-*- ## generated by cyg-apt, changes will be lost. ## Define Cygwin/Mingw flavour %(cygcomment)starget=${target-cygwin} %(mincomment)starget=${target-mingw} ## Everything is relative to mknetrel_root, netrel_root or cygwin_root ## the mknetrel defaults are ## : ${mknetrel_root="$(cd $(dirname $0)/.. && pwd)"} ## : ${netrel_root="/netrel"} ## : ${cygwin_root="/cygwin"} #: ${mknetrel_root="$HOME/netrel"} #: ${netrel_root="$HOME/$target"} #: ${cygwin_root="$HOME/$target/root"} #: ${mknetrel_root="$(cd $(dirname $0)/.. && pwd)"} #: ${netrel_root="$mknetrel_root/$target"} #: ${cygwin_root="$mknetrel_root/$target/root"} : ${mknetrel_root="%(mknetrel_root)s"} : ${netrel_root="%(netrel_root)s"} : ${cygwin_root="%(cygwin_root)s"} case "$(uname -s)" in CYGWIN*) iscygwin () { :; } ;; *) iscygwin () { return 1; } ;; esac iscygwin || . $mknetrel_root/mknetrel/crossvars ''' % vars ()) def do_unpack (): # find ball ball = get_ball () # untar capture list # tarfile #pipe = os.popen ('tar -C "%s" -xjvf "%s"' % (CWD, ball), 'r') global packagename basename = os.path.basename (ball) packagename = re.sub ('(-src)*\.tar\.(bz2|gz)', '', basename) if os.path.exists ('%s/%s' % (SRC, packagename)): return pipe = os.popen ('tar -C "%s" -xjvf "%s"' % (SRC, ball), 'r') lst = map (string.strip, pipe.readlines ()) if pipe.close (): raise 'urg1' print ('%s/%s' % (SRC, packagename)) if not os.path.exists ('%s/%s' % (SRC, packagename)): raise 'urg2' def do_build (): src = '%s/%s' % (SRC, packagename) if not os.path.exists (src): raise 'urg' m = re.match ('^(.*)-([0-9]*)$', packagename) if not m: raise 'urg' namever = m.group (1) package = split_ball (packagename) name = package[0] #namever = name + '-' + string.join (package[1][1:-1], '.') pbuild = package[1][-1] # ugh: mknetrel should source /cygwin/mknetrel # copy to mknetrel's EXTRA dir for now cygwin = src + '/' + target script = cygwin + '/mknetrel' if os.path.exists (script): shutil.copy (script, '%s/%s' % (EXTRA, namever)) os.system ('mknetrel %s' % namever) def build (): '''build package from source in CWD''' global packagename if not packagename: packagename = os.path.basename (CWD) do_build () def source (): '''download, build and install''' global packagename # let's not do dependencies #for packagename in missing.keys (): global INSTALL INSTALL = 'source' for packagename in files[1:]: download () for packagename in files[1:]: do_unpack () if download_p: sys.exit (0) for packagename in files[1:]: do_build () sys.exit (0) def find (): '''package containing file''' global packagename regexp = re.sub ('^%s/' % ROOT, '/', packagename) hits = [] for packagename in psort (installed[0].keys ()): for i in get_filelist (): if re.search (regexp, '/%s' % i): hits.append ('%s: /%s' % (packagename, i)) print (string.join (hits, '\n')) if command == 'setup': setup () sys.exit (0) if command == 'update': update () sys.exit (0) for i in (installed_db, setup_ini): if not os.path.isfile (i): usage () sys.stderr.write ('\n') sys.stderr.write ('error: %s no such file\n' % i) sys.stderr.write ('error: run %(basename)s setup?\n' % vars ()) sys.exit (2) get_setup_ini () get_installed () if command and command in __main__.__dict__: __main__.__dict__[command] ()