diff options
Diffstat (limited to 'bitbake/lib/bb/fetch')
| -rw-r--r-- | bitbake/lib/bb/fetch/__init__.py | 35 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch/bk.py | 40 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch/cvs.py | 20 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch/git.py | 95 | ||||
| -rw-r--r-- | bitbake/lib/bb/fetch/svn.py | 27 |
5 files changed, 83 insertions, 134 deletions
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py index da5b10c4b..0515f2a5e 100644 --- a/bitbake/lib/bb/fetch/__init__.py +++ b/bitbake/lib/bb/fetch/__init__.py @@ -158,18 +158,47 @@ class Fetch(object): return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1 ) getSRCDate = staticmethod(getSRCDate) -#if __name__ == "__main__": + def try_mirror(d, tarfn): + """ + Try to use a mirrored version of the sources. We do this + to avoid massive loads on foreign cvs and svn servers. + This method will be used by the different fetcher + implementations. + + d Is a bb.data instance + tarfn is the name of the tarball + """ + tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn) + if os.access(tarpath, os.R_OK): + return True + + pn = data.getVar('PN', d, True) + src_tarball_stash = None + if pn: + src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split() + + for stash in src_tarball_stash: + fetchcmd = data.getVar("FETCHCOMMAND_mirror", d, True) or data.getVar("FETCHCOMMAND_wget", d, True) + uri = stash + tarfn + bb.note("fetch " + uri) + fetchcmd = fetchcmd.replace("${URI}", uri) + ret = os.system(fetchcmd) + if ret == 0: + bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn) + return True + return False + try_mirror = staticmethod(try_mirror) -import bk import cvs import git import local import svn import wget +import svk -methods.append(bk.Bk()) methods.append(cvs.Cvs()) methods.append(git.Git()) methods.append(local.Local()) methods.append(svn.Svn()) methods.append(wget.Wget()) +methods.append(svk.Svk()) diff --git a/bitbake/lib/bb/fetch/bk.py b/bitbake/lib/bb/fetch/bk.py deleted file mode 100644 index 6bd6c018f..000000000 --- a/bitbake/lib/bb/fetch/bk.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -""" -BitBake 'Fetch' implementations - -Classes for obtaining upstream sources for the -BitBake build tools. - -Copyright (C) 2003, 2004 Chris Larson - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; either version 2 of the License, or (at your option) any later -version. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., 59 Temple -Place, Suite 330, Boston, MA 02111-1307 USA. - -Based on functions from the base bb module, Copyright 2003 Holger Schurig -""" - -import os, re -import bb -from bb import data -from bb.fetch import Fetch - -class Bk(Fetch): - def supports(url, d): - """Check to see if a given url can be fetched via bitkeeper. - Expects supplied url in list form, as outputted by bb.decodeurl(). - """ - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) - return type in ['bk'] - supports = staticmethod(supports) diff --git a/bitbake/lib/bb/fetch/cvs.py b/bitbake/lib/bb/fetch/cvs.py index 461a2f50f..10ec700dc 100644 --- a/bitbake/lib/bb/fetch/cvs.py +++ b/bitbake/lib/bb/fetch/cvs.py @@ -133,21 +133,9 @@ class Cvs(Fetch): bb.debug(1, "%s already exists, skipping cvs checkout." % tarfn) continue - pn = data.getVar('PN', d, 1) - cvs_tarball_stash = None - if pn: - cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1) - if cvs_tarball_stash == None: - cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1) - if cvs_tarball_stash: - fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1) - uri = cvs_tarball_stash + tarfn - bb.note("fetch " + uri) - fetchcmd = fetchcmd.replace("${URI}", uri) - ret = os.system(fetchcmd) - if ret == 0: - bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn) - continue + # try to use the tarball stash + if Fetch.try_mirror(d, tarfn): + continue if date: options.append("-D %s" % date) @@ -194,7 +182,7 @@ class Cvs(Fetch): bb.debug(1, "Running %s" % cvscmd) myret = os.system(cvscmd) - if myret != 0: + if myret != 0 or not os.access(moddir, os.R_OK): try: os.rmdir(moddir) except OSError: diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py index 296b92639..439d52218 100644 --- a/bitbake/lib/bb/fetch/git.py +++ b/bitbake/lib/bb/fetch/git.py @@ -58,6 +58,28 @@ def gettag(parm): return tag +def getprotocol(parm): + if 'protocol' in parm: + proto = parm['protocol'] + else: + proto = "" + if not proto: + proto = "rsync" + + return proto + +def localfile(url, d): + """Return the filename to cache the checkout in""" + (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) + + #if user sets localpath for file, use it instead. + if "localpath" in parm: + return parm["localpath"] + + tag = gettag(parm) + + return data.expand('git_%s%s_%s.tar.gz' % (host, path.replace('/', '.'), tag), d) + class Git(Fetch): """Class to fetch a module or modules from git repositories""" def supports(url, d): @@ -69,17 +91,8 @@ class Git(Fetch): supports = staticmethod(supports) def localpath(url, d): - (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d)) - - #if user sets localpath for file, use it instead. - if "localpath" in parm: - return parm["localpath"] - tag = gettag(parm) - - localname = data.expand('git_%s%s_%s.tar.gz' % (host, path.replace('/', '.'), tag), d) - - return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s' % (localname), d)) + return os.path.join(data.getVar("DL_DIR", d, 1), localfile(url, d)) localpath = staticmethod(localpath) @@ -92,10 +105,12 @@ class Git(Fetch): (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, d)) tag = gettag(parm) + proto = getprotocol(parm) gitsrcname = '%s%s' % (host, path.replace('/', '.')) - repofile = os.path.join(data.getVar("DL_DIR", d, 1), 'git_%s.tar.gz' % (gitsrcname)) + repofilename = 'git_%s.tar.gz' % (gitsrcname) + repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename) repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) coname = '%s' % (tag) @@ -103,63 +118,37 @@ class Git(Fetch): cofile = self.localpath(loc, d) - # Always update to current if tag=="master" - #if os.access(cofile, os.R_OK) and (tag != "master"): - if os.access(cofile, os.R_OK): - bb.debug(1, "%s already exists, skipping git checkout." % cofile) + # tag=="master" must always update + if (tag != "master") and Fetch.try_mirror(d, localfile(loc, d)): + bb.debug(1, "%s already exists (or was stashed). Skipping git checkout." % cofile) continue -# Still Need to add GIT_TARBALL_STASH Support... -# pn = data.getVar('PN', d, 1) -# cvs_tarball_stash = None -# if pn: -# cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1) -# if cvs_tarball_stash == None: -# cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1) -# if cvs_tarball_stash: -# fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1) -# uri = cvs_tarball_stash + tarfn -# bb.note("fetch " + uri) -# fetchcmd = fetchcmd.replace("${URI}", uri) -# ret = os.system(fetchcmd) -# if ret == 0: -# bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn) -# continue - - #if os.path.exists(repodir): - #prunedir(repodir) - - bb.mkdirhier(repodir) - os.chdir(repodir) - - #print("Changing to %s" % repodir) + if not os.path.exists(repodir): + if Fetch.try_mirror(d, repofilename): + bb.mkdirhier(repodir) + os.chdir(repodir) + rungitcmd("tar -xzf %s" % (repofile),d) + else: + rungitcmd("git clone %s://%s%s %s" % (proto, host, path, repodir),d) - if os.access(repofile, os.R_OK): - rungitcmd("tar -xzf %s" % (repofile),d) - else: - rungitcmd("git clone rsync://%s%s %s" % (host, path, repodir),d) - - rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (host, path, os.path.join(repodir, ".git", "")),d) - - #print("Changing to %s" % repodir) os.chdir(repodir) - rungitcmd("git pull rsync://%s%s" % (host, path),d) + rungitcmd("git pull %s://%s%s" % (proto, host, path),d) + rungitcmd("git pull --tags %s://%s%s" % (proto, host, path),d) + # old method of downloading tags + #rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (host, path, os.path.join(repodir, ".git", "")),d) - #print("Changing to %s" % repodir) os.chdir(repodir) + bb.note("Creating tarball of git repository") rungitcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ),d) if os.path.exists(codir): prunedir(codir) - #print("Changing to %s" % repodir) bb.mkdirhier(codir) os.chdir(repodir) rungitcmd("git read-tree %s" % (tag),d) - rungitcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")),d) - #print("Changing to %s" % codir) os.chdir(codir) + bb.note("Creating tarball of git checkout") rungitcmd("tar -czf %s %s" % (cofile, os.path.join(".", "*") ),d) - diff --git a/bitbake/lib/bb/fetch/svn.py b/bitbake/lib/bb/fetch/svn.py index ac5eebf5c..6e3a9277a 100644 --- a/bitbake/lib/bb/fetch/svn.py +++ b/bitbake/lib/bb/fetch/svn.py @@ -98,20 +98,14 @@ class Svn(Fetch): date = Fetch.getSRCDate(d) - if "method" in parm: - method = parm["method"] - else: - method = "pserver" - if "proto" in parm: proto = parm["proto"] else: proto = "svn" svn_rsh = None - if method == "ext": - if "rsh" in parm: - svn_rsh = parm["rsh"] + if proto == "svn+ssh" and "rsh" in parm: + svn_rsh = parm["rsh"] tarfn = data.expand('%s_%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, path.replace('/', '.'), revision, date), localdata) data.setVar('TARFILES', dlfile, localdata) @@ -122,24 +116,13 @@ class Svn(Fetch): bb.debug(1, "%s already exists, skipping svn checkout." % tarfn) continue - svn_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1) - if svn_tarball_stash: - fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1) - uri = svn_tarball_stash + tarfn - bb.note("fetch " + uri) - fetchcmd = fetchcmd.replace("${URI}", uri) - ret = os.system(fetchcmd) - if ret == 0: - bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn) - continue + # try to use the tarball stash + if Fetch.try_mirror(d, tarfn): + continue olddir = os.path.abspath(os.getcwd()) os.chdir(data.expand(dldir, localdata)) -# setup svnroot -# svnroot = ":" + method + ":" + user -# if pswd: -# svnroot += ":" + pswd svnroot = host + path data.setVar('SVNROOT', svnroot, localdata) |