summaryrefslogtreecommitdiff
path: root/bitbake
diff options
context:
space:
mode:
authorJoshua Lock <josh@linux.intel.com>2010-02-01 16:56:16 +0000
committerJoshua Lock <josh@linux.intel.com>2010-02-04 00:18:29 +0000
commit0737552c1de18c64db8a5a52248a3e7b217f30aa (patch)
treed556e058c5e3b85dde0b3c13f5e9e25e472a5ed8 /bitbake
parent0b52537f5ce39a6025a2a8de381792accc333b8e (diff)
downloadopenembedded-core-0737552c1de18c64db8a5a52248a3e7b217f30aa.tar.gz
openembedded-core-0737552c1de18c64db8a5a52248a3e7b217f30aa.tar.bz2
openembedded-core-0737552c1de18c64db8a5a52248a3e7b217f30aa.tar.xz
openembedded-core-0737552c1de18c64db8a5a52248a3e7b217f30aa.zip
bitbake: unify mirror support and make it independant of the fetcher
This patch serves two purposes. Firstly it unifies the concept of mirrors into PREMIRRORS and MIRRORS. PREMIRRORS are tried before the SRC_URI defined in the recipe whereas MIRRORS are tried only if that fails. The tarball stash was conceptually inline with a PREMIRROR only with special handling within the wget fetcher and therefore only worked with certain fetch types. Secondly the patch removes the need for individual fetch implementations to worry about mirror handling. With this patch, the base fetch implementation will first try to use a PREMIRROR to fetch the desired object, if this fails the native fetch method for the object will be tried and if this fails will try to fetch a copy from one of the MIRRORS. Signed-off-by: Joshua Lock <josh@linux.intel.com>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/fetch/__init__.py94
-rw-r--r--bitbake/lib/bb/fetch/bzr.py5
-rw-r--r--bitbake/lib/bb/fetch/cvs.py5
-rw-r--r--bitbake/lib/bb/fetch/git.py9
-rw-r--r--bitbake/lib/bb/fetch/hg.py5
-rw-r--r--bitbake/lib/bb/fetch/osc.py5
-rw-r--r--bitbake/lib/bb/fetch/perforce.py5
-rw-r--r--bitbake/lib/bb/fetch/svk.py3
-rw-r--r--bitbake/lib/bb/fetch/svn.py5
-rw-r--r--bitbake/lib/bb/fetch/wget.py16
10 files changed, 53 insertions, 99 deletions
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py
index ab4658bc3..ccb60de59 100644
--- a/bitbake/lib/bb/fetch/__init__.py
+++ b/bitbake/lib/bb/fetch/__init__.py
@@ -185,7 +185,18 @@ def go(d, urls = None):
pass
bb.utils.unlockfile(lf)
continue
- m.go(u, ud, d)
+
+ # First try fetching uri, u, from PREMIRRORS
+ mirrors = [ i.split() for i in (bb.data.getVar('PREMIRRORS', d, 1) or "").split('\n') if i ]
+ if not try_mirrors(d, u, mirrors):
+ # Next try fetching from the original uri, u
+ try:
+ m.go(u, ud, d)
+ except:
+ # Finally, try fetching uri, u, from MIRRORS
+ mirrors = [ i.split() for i in (bb.data.getVar('MIRRORS', d, 1) or "").split('\n') if i ]
+ try_mirrors (d, u, mirrors)
+
if ud.localfile:
if not m.forcefetch(u, ud, d):
Fetch.write_md5sum(u, ud, d)
@@ -332,6 +343,44 @@ def runfetchcmd(cmd, d, quiet = False):
return output
+def try_mirrors(d, uri, mirrors):
+ """
+ Try to use a mirrored version of the sources.
+ This method will be automatically called before the fetchers go.
+
+ d Is a bb.data instance
+ uri is the original uri we're trying to download
+ mirrors is the list of mirrors we're going to try
+ """
+ fpath = os.path.join(data.getVar("DL_DIR", d, 1), os.path.basename(uri))
+ if os.access(fpath, os.R_OK):
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % fpath)
+ return True
+
+ ld = d.createCopy()
+ for (find, replace) in mirrors:
+ newuri = uri_replace(uri, find, replace, ld)
+ if newuri != uri:
+ try:
+ ud = FetchData(newuri, ld)
+ except bb.fetch.NoMethodError:
+ bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url)
+ continue
+
+ ud.setup_localpath(ld)
+
+ try:
+ ud.method.go(newuri, ud, ld)
+ return True
+ except (bb.fetch.MissingParameterError,
+ bb.fetch.FetchError,
+ bb.fetch.MD5SumError):
+ import sys
+ (type, value, traceback) = sys.exc_info()
+ bb.msg.debug(2, bb.msg.domain.Fetcher, "Mirror fetch failure: %s" % value)
+ return False
+
+
class FetchData(object):
"""
A class which represents the fetcher state for a given URI.
@@ -489,49 +538,6 @@ class Fetch(object):
localcount_internal_helper = staticmethod(localcount_internal_helper)
- def try_mirror(d, tarfn):
- """
- Try to use a mirrored version of the sources. We do this
- to avoid massive loads on foreign cvs and svn servers.
- This method will be used by the different fetcher
- implementations.
-
- d Is a bb.data instance
- tarfn is the name of the tarball
- """
- tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn)
- if os.access(tarpath, os.R_OK):
- bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn)
- return True
-
- pn = data.getVar('PN', d, True)
- src_tarball_stash = None
- if pn:
- src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split()
-
- ld = d.createCopy()
- for stash in src_tarball_stash:
- url = stash + tarfn
- try:
- ud = FetchData(url, ld)
- except bb.fetch.NoMethodError:
- bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url)
- continue
-
- ud.setup_localpath(ld)
-
- try:
- ud.method.go(url, ud, ld)
- return True
- except (bb.fetch.MissingParameterError,
- bb.fetch.FetchError,
- bb.fetch.MD5SumError):
- import sys
- (type, value, traceback) = sys.exc_info()
- bb.msg.debug(2, bb.msg.domain.Fetcher, "Tarball stash fetch failure: %s" % value)
- return False
- try_mirror = staticmethod(try_mirror)
-
def verify_md5sum(ud, got_sum):
"""
Verify the md5sum we wanted with the one we got
diff --git a/bitbake/lib/bb/fetch/bzr.py b/bitbake/lib/bb/fetch/bzr.py
index b27fb63d0..c6e33c334 100644
--- a/bitbake/lib/bb/fetch/bzr.py
+++ b/bitbake/lib/bb/fetch/bzr.py
@@ -91,11 +91,6 @@ class Bzr(Fetch):
def go(self, loc, ud, d):
"""Fetch url"""
- # try to use the tarball stash
- if Fetch.try_mirror(d, ud.localfile):
- bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping bzr checkout." % ud.localpath)
- return
-
if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
bzrcmd = self._buildbzrcommand(ud, d, "update")
bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Update %s" % loc)
diff --git a/bitbake/lib/bb/fetch/cvs.py b/bitbake/lib/bb/fetch/cvs.py
index 90a006500..443f52131 100644
--- a/bitbake/lib/bb/fetch/cvs.py
+++ b/bitbake/lib/bb/fetch/cvs.py
@@ -77,11 +77,6 @@ class Cvs(Fetch):
def go(self, loc, ud, d):
- # try to use the tarball stash
- if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
- bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping cvs checkout." % ud.localpath)
- return
-
method = "pserver"
if "method" in ud.parm:
method = ud.parm["method"]
diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py
index 9430582d5..10b396bb0 100644
--- a/bitbake/lib/bb/fetch/git.py
+++ b/bitbake/lib/bb/fetch/git.py
@@ -82,10 +82,6 @@ class Git(Fetch):
def go(self, loc, ud, d):
"""Fetch url"""
- if Fetch.try_mirror(d, ud.localfile):
- bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
- return
-
if ud.user:
username = ud.user + '@'
else:
@@ -97,11 +93,12 @@ class Git(Fetch):
codir = os.path.join(ud.clonedir, coname)
if not os.path.exists(ud.clonedir):
- if Fetch.try_mirror(d, ud.mirrortarball):
+ try:
+ Fetch.try_mirrors(ud.mirrortarball)
bb.mkdirhier(ud.clonedir)
os.chdir(ud.clonedir)
runfetchcmd("tar -xzf %s" % (repofile), d)
- else:
+ except:
runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.clonedir), d)
os.chdir(ud.clonedir)
diff --git a/bitbake/lib/bb/fetch/hg.py b/bitbake/lib/bb/fetch/hg.py
index 08cb61fc2..d0756382f 100644
--- a/bitbake/lib/bb/fetch/hg.py
+++ b/bitbake/lib/bb/fetch/hg.py
@@ -116,11 +116,6 @@ class Hg(Fetch):
def go(self, loc, ud, d):
"""Fetch url"""
- # try to use the tarball stash
- if Fetch.try_mirror(d, ud.localfile):
- bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping hg checkout." % ud.localpath)
- return
-
bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
diff --git a/bitbake/lib/bb/fetch/osc.py b/bitbake/lib/bb/fetch/osc.py
index 2c34caf6c..548dd9d07 100644
--- a/bitbake/lib/bb/fetch/osc.py
+++ b/bitbake/lib/bb/fetch/osc.py
@@ -91,11 +91,6 @@ class Osc(Fetch):
Fetch url
"""
- # Try to use the tarball stash
- if Fetch.try_mirror(d, ud.localfile):
- bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping osc checkout." % ud.localpath)
- return
-
bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK):
diff --git a/bitbake/lib/bb/fetch/perforce.py b/bitbake/lib/bb/fetch/perforce.py
index 394f5a225..e2c342108 100644
--- a/bitbake/lib/bb/fetch/perforce.py
+++ b/bitbake/lib/bb/fetch/perforce.py
@@ -124,11 +124,6 @@ class Perforce(Fetch):
Fetch urls
"""
- # try to use the tarball stash
- if Fetch.try_mirror(d, ud.localfile):
- bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping perforce checkout." % ud.localpath)
- return
-
(host,depot,user,pswd,parm) = Perforce.doparse(loc, d)
if depot.find('/...') != -1:
diff --git a/bitbake/lib/bb/fetch/svk.py b/bitbake/lib/bb/fetch/svk.py
index 120dad9d4..a17ac04d2 100644
--- a/bitbake/lib/bb/fetch/svk.py
+++ b/bitbake/lib/bb/fetch/svk.py
@@ -62,9 +62,6 @@ class Svk(Fetch):
def go(self, loc, ud, d):
"""Fetch urls"""
- if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
- return
-
svkroot = ud.host + ud.path
svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module)
diff --git a/bitbake/lib/bb/fetch/svn.py b/bitbake/lib/bb/fetch/svn.py
index eef9862a8..ba9f6ab10 100644
--- a/bitbake/lib/bb/fetch/svn.py
+++ b/bitbake/lib/bb/fetch/svn.py
@@ -136,11 +136,6 @@ class Svn(Fetch):
def go(self, loc, ud, d):
"""Fetch url"""
- # try to use the tarball stash
- if Fetch.try_mirror(d, ud.localfile):
- bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping svn checkout." % ud.localpath)
- return
-
bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
diff --git a/bitbake/lib/bb/fetch/wget.py b/bitbake/lib/bb/fetch/wget.py
index fd93c7ec4..ae1c6ad13 100644
--- a/bitbake/lib/bb/fetch/wget.py
+++ b/bitbake/lib/bb/fetch/wget.py
@@ -30,7 +30,6 @@ import bb
from bb import data
from bb.fetch import Fetch
from bb.fetch import FetchError
-from bb.fetch import uri_replace
class Wget(Fetch):
"""Class to fetch urls via 'wget'"""
@@ -105,24 +104,9 @@ class Wget(Fetch):
data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata)
data.update_data(localdata)
- premirrors = [ i.split() for i in (data.getVar('PREMIRRORS', localdata, 1) or "").split('\n') if i ]
- for (find, replace) in premirrors:
- newuri = uri_replace(uri, find, replace, d)
- if newuri != uri:
- if fetch_uri(newuri, ud, localdata):
- return True
-
if fetch_uri(uri, ud, localdata):
return True
- # try mirrors
- mirrors = [ i.split() for i in (data.getVar('MIRRORS', localdata, 1) or "").split('\n') if i ]
- for (find, replace) in mirrors:
- newuri = uri_replace(uri, find, replace, d)
- if newuri != uri:
- if fetch_uri(newuri, ud, localdata):
- return True
-
raise FetchError(uri)