From 7d63e57fdaa5e54684fa366dd9d6e9149100f569 Mon Sep 17 00:00:00 2001
From: Pawel Szostek <pawel.szostek@gmail.com>
Date: Thu, 19 May 2011 18:11:41 +0200
Subject: [PATCH] Fixed recursive fetching.
---
synthesis/configparser.py | 55 +++++++----
synthesis/fetch.py | 181 +++++++++++++++++++++++++++++-------
synthesis/helper_classes.py | 6 +-
synthesis/module.py | 52 ++---------
synthesis/srcfile.py | 3 +
5 files changed, 196 insertions(+), 101 deletions(-)
diff --git a/synthesis/configparser.py b/synthesis/configparser.py
index 4c094022..dd426a08 100644
--- a/synthesis/configparser.py
+++ b/synthesis/configparser.py
@@ -124,44 +124,62 @@ class ConfigParser(object):
if not isinstance(description, basestring):
raise ValueError("Description should be a string!")
self.description = description
- self.options = {}
+ self.options = []
self.arbitrary_code = ""
+ def __setitem__(self, name, value):
+ if name in self.__names():
+ filter(lambda x: x.name == name, self.options)[0] = value
+ else:
+ self.options.append(value)
+
+ def __getitem__(self, name):
+ if name in self.__names():
+ return filter(lambda x: x != None and x.name == name, self.options)[0]
+ else:
+ raise RuntimeException("No such option as " + str(name))
+
def help(self):
p.rawprint("Variables available in a manifest:")
- for key in self.options:
- opt = self.options[key]
- line = ' {0:10}; {1:15}; {2:40}{3}{4:10}'
+ for opt in self.options:
+ if opt == None:
+ p.rawprint("")
+ continue
+
+ line = ' {0:15}; {1:29}; {2:45}{3}{4:10}'
try:
tmp_def = opt.default
if tmp_def == "":
tmp_def = '""'
- line = line.format(key, str(opt.types), opt.help,', default=', tmp_def)
+ line = line.format(opt.name, str(opt.types), opt.help,', default=', tmp_def)
except AttributeError: #no default value
line = line.format(key, str(opt.types), opt.help, "","")
p.rawprint(line)
def add_option(self, name, **others):
- if name in self.options:
+ if name in self.__names():
raise ValueError("Option already added: " + name)
- self.options[name] = ConfigParser.Option(name, **others)
+ self.options.append(ConfigParser.Option(name, **others))
def add_type(self, name, type):
- if name not in self.options:
+ if name not in self.__names():
raise RuntimeError("Can't add type to a non-existing option")
- self.options[name].add_type(type_obj=type)
+ self[name].add_type(type)
+
+ def add_delimiter(self):
+ self.options.append(None)
def add_allowed_key(self, name, key):
if not isinstance(key, basestring):
raise ValueError("Allowed key must be a string")
try:
- self.options[name].allowed_keys.append(key)
+ self[name].allowed_keys.append(key)
except AttributeError:
- if type(dict()) not in self.options[name].types:
+ if type(dict()) not in self[name].types:
raise RuntimeError("Allowing a key makes sense for dictionaries only")
- self.options[name].allowed_keys = [key]
+ self[name].allowed_keys = [key]
- self.options[name].allowed_keys.append(key)
+ self[name].allowed_keys.append(key)
def add_config_file(self, config_file):
try:
@@ -178,6 +196,9 @@ class ConfigParser(object):
def add_arbitrary_code(self, code):
self.arbitrary_code += code + '\n'
+ def __names(self):
+ return [o.name for o in self.options if o != None]
+
def parse(self):
options = {}
ret = {}
@@ -212,24 +233,24 @@ class ConfigParser(object):
for opt_name, val in list(options.items()): #check delivered options
if opt_name.startswith('__'):
continue
- if opt_name not in self.options:
+ if opt_name not in self.__names():
if opt_name in arbitrary_options:
continue
else:
raise NameError("Unrecognized option: " + opt_name)
- opt = self.options[opt_name]
+ opt = self[opt_name]
if type(val) not in opt.types:
raise RuntimeError("Given option: "+str(type(val))+" doesn't match specified types:"+str(opt.types))
ret[opt_name] = val
if type(val) == type(dict()):
try:
for key in val:
- if key not in self.options[opt_name].allowed_keys:
+ if key not in self[opt_name].allowed_keys:
raise RuntimeError("Encountered unallowed key: " +key+ " for options '"+opt_name+"'")
except AttributeError: #no allowed_keys member - don't perform any check
pass
- for name, opt in self.options.items(): #set values for not listed items with defaults
+ for opt in self.options: #set values for not listed items with defaults
try:
if opt.name not in ret:
ret[opt.name] = opt.default
diff --git a/synthesis/fetch.py b/synthesis/fetch.py
index b27368bf..b493fefd 100644
--- a/synthesis/fetch.py
+++ b/synthesis/fetch.py
@@ -6,45 +6,31 @@ import global_mod
import path
class ModuleFetcher:
-
def __init__(self):
pass
def fetch_single_module(self, module):
- involved_modules = []
+ new_modules = []
p.vprint("Fetching manifest: " + str(module.manifest))
- if(module.source == "local"):
+ if module.source == "local":
p.vprint("ModPath: " + module.path);
- module.parse_manifest()
-
- if module.root_module != None:
- root_module = module.root_module
- p.vprint("Encountered root manifest: " + str(root_module))
- new_modules = self.fetch_recursively(root_module)
- involved_modules.extend(new_modules)
-
- for module in module.svn:
+ if module.source == "svn":
p.vprint("[svn] Fetching to " + module.fetchto)
self.__fetch_from_svn(module)
- module.source = "local"
- module.isparsed = False
- p.vprint("[svn] Local path " + module.path)
- involved_modules.append(module)
-
- for module in module.git:
+ if module.source == "git":
p.vprint("[git] Fetching to " + module.fetchto)
self.__fetch_from_git(module)
- module.source = "local"
- module.isparsed = False
- module.manifest = module.search_for_manifest();
- p.vprint("[git] Local path " + module.path);
- involved_modules.append(module)
- for module in module.local:
- involved_modules.append(module)
+ module.parse_manifest()
- return involved_modules
+ if module.root_module != None:
+ p.vprint("Encountered root manifest: " + str(root_module))
+ new_modules.append(module.root_module)
+
+ new_modules.extend(module.svn)
+ new_modules.extend(module.git)
+ return new_modules
def __fetch_from_svn(self, module):
fetchto = module.fetchto
@@ -135,11 +121,128 @@ class ModuleFetcher:
return ret
class ModulePool(list):
+ class ModuleFetcher:
+ def __init__(self):
+ pass
+
+ def fetch_single_module(self, module):
+ new_modules = []
+ p.vprint("Fetching manifest: " + str(module.manifest))
+
+ if module.source == "local":
+ print "local module in fetching"
+ p.vprint("ModPath: " + module.path);
+ if module.source == "svn":
+ p.vprint("[svn] Fetching to " + module.fetchto)
+ self.__fetch_from_svn(module)
+ module.manifest = module.search_for_manifest()
+ if module.source == "git":
+ p.vprint("[git] Fetching to " + module.fetchto)
+ self.__fetch_from_git(module)
+ module.manifest = module.search_for_manifest()
+
+ module.parse_manifest()
+
+ if module.root_module != None:
+ p.vprint("Encountered root manifest: " + str(root_module))
+ new_modules.append(module.root_module)
+
+ new_modules.extend(module.svn)
+ new_modules.extend(module.git)
+ return new_modules
+
+ def __fetch_from_svn(self, module):
+ fetchto = module.fetchto
+ if not os.path.exists(fetchto):
+ os.mkdir(fetchto)
+
+ cur_dir = os.getcwd()
+ os.chdir(fetchto)
+ url, rev = __parse_repo_url(module.url)
+
+ basename = path.url_basename(url)
+
+ cmd = "svn checkout {0} " + basename
+ if rev:
+ cmd = cmd.format(url + '@' + rev)
+ else:
+ cmd = cmd.format(url)
+
+ rval = True
+
+ p.vprint(cmd)
+ if os.system(cmd) != 0:
+ rval = False
+ os.chdir(cur_dir)
+
+ module.isfetched = True
+ module.revision = rev
+ module.path = os.path.join(fetchto, basename)
+ return rval
+
+ def __fetch_from_git(self, module):
+ fetchto = module.fetchto
+ if not os.path.exists(fetchto):
+ os.mkdir(fetchto)
+
+ cur_dir = os.getcwd()
+ os.chdir(fetchto)
+ url, rev = self.__parse_repo_url(module.url)
+
+ basename = path.url_basename(url)
+
+ if basename.endswith(".git"):
+ basename = basename[:-4] #remove trailing .git
+
+ if not os.path.exists(os.path.join(fetchto, basename)):
+ update_only = False
+ else:
+ update_only = True
+
+ if update_only:
+ cmd = "git --git-dir="+basename+"/.git pull"
+ else:
+ cmd = "git clone " + url
+
+ rval = True
+
+ p.vprint(cmd)
+ if os.system(cmd) != 0:
+ rval = False
+
+ if rev and rval:
+ os.chdir(basename)
+ cmd = "git checkout " + revision
+ p.vprint(cmd)
+ if os.system(cmd) != 0:
+ rval = False
+
+ os.chdir(cur_dir)
+ module.isfetched = True
+ module.revision = rev
+ module.path = os.path.join(fetchto, basename)
+ return rval
+ #end class ModuleFetcher
+ def __parse_repo_url(self, url) :
+ """
+ Check if link to a repo seems to be correct. Filter revision number
+ """
+ import re
+ url_pat = re.compile("[ \t]*([^ \t]+)[ \t]*(@[ \t]*(.+))?[ \t]*")
+ url_match = re.match(url_pat, url)
+
+ if url_match == None:
+ p.echo("Not a correct repo url: {0}. Skipping".format(url))
+ if url_match.group(3) != None: #there is a revision given
+ ret = (url_match.group(1), url_match.group(3))
+ else:
+ ret = (url_match.group(1), None)
+ return ret
def __init__(self, top_module):
self.top_module = top_module
self.modules = []
- self.add(module=top_module)
-
+ self.add(new_module=top_module)
+
def __iter__(self):
return self.modules.__iter__()
@@ -155,15 +258,19 @@ class ModulePool(list):
def __str__(self):
return str([str(m) for m in self.modules])
- def add(self, module):
- from module import Module
- if not isinstance(module, Module):
- raise RuntimeError("Expecting a Module instance")
+ def __contains(self, module):
for mod in self.modules:
if mod.url == module.url:
return False
- self.modules.append(module)
- for m in module.git + module.svn + module.local:
+ return True
+
+ def add(self, new_module):
+ from module import Module
+ if not isinstance(new_module, Module):
+ raise RuntimeError("Expecting a Module instance")
+ if self.__contains(new_module):
+ return
+ for m in new_module.isfetched:
self.add(m)
return True
@@ -173,10 +280,12 @@ class ModulePool(list):
while len(fetch_queue) > 0:
cur_mod = fetch_queue.pop()
+ self.add(cur_mod)
+ print "<<<<<<" + cur_mod.url
new_modules = fetcher.fetch_single_module(cur_mod)
+
for mod in new_modules:
- ret = self.add(mod)
- if ret == True:
+ if self.__contains(mod):
fetch_queue.append(mod)
else:
pass
diff --git a/synthesis/helper_classes.py b/synthesis/helper_classes.py
index 62391742..cd471615 100644
--- a/synthesis/helper_classes.py
+++ b/synthesis/helper_classes.py
@@ -31,19 +31,20 @@ class ManifestParser(ConfigParser):
self.add_option('root_module', default=None, help="Path to root module for currently parsed", type='')
self.add_option('name', default=None, help="Name of the folder at remote synthesis machine", type='')
-
+ self.add_delimiter()
self.add_option('syn_device', default=None, help = "Target FPGA device", type = '');
self.add_option('syn_grade', default=None, help = "Speed grade of target FPGA", type = '');
self.add_option('syn_package', default=None, help = "Package variant of target FPGA", type = '');
self.add_option('syn_top', default=None, help = "Top level module for synthesis", type = '');
self.add_option('syn_project', default=None, help = "Vendor flow project file", type = '');
-
+ self.add_delimiter()
self.add_option('vsim_opt', default="", help="Additional options for vsim", type='')
self.add_option('vcom_opt', default="", help="Additional options for vcom", type='')
self.add_option('vlog_opt', default="", help="Additional options for vlog", type='')
self.add_option('vmap_opt', default="", help="Additional options for vmap", type='')
+ self.add_delimiter()
self.add_option('modules', default={}, help="List of local modules", type={})
self.add_option('target', default=None, help="Target architecture for synthesis", type='')
@@ -51,6 +52,7 @@ class ManifestParser(ConfigParser):
self.add_allowed_key('modules', key="git")
self.add_allowed_key('modules', key="local")
+ #self.add_delimiter()
self.add_option('library', default="work",
help="Destination library for module's VHDL files", type="")
self.add_option('files', default=[], help="List of files from the current module", type='')
diff --git a/synthesis/module.py b/synthesis/module.py
index 4d74b4d8..6ab5e439 100644
--- a/synthesis/module.py
+++ b/synthesis/module.py
@@ -148,10 +148,14 @@ class Module(object):
return sth
def parse_manifest(self):
+ print ">>>>>Parsing manifest " + self.url
if self.isparsed == True:
return
if self.isfetched == False:
return
+ if self.manifest == None:
+ self.manifest = self.__search_for_manifest()
+ print "MMM"+str(self.manifest)
manifest_parser = ManifestParser()
if(self.parent != None):
@@ -259,6 +263,7 @@ class Module(object):
else:
self.svn = []
+ print self.svn
if "git" in opt_map["modules"]:
opt_map["modules"]["git"] = self.__make_list(opt_map["modules"]["git"])
git = []
@@ -343,49 +348,4 @@ class Module(object):
for m in modules:
f_set.add(m.fileset);
- return f_set
-#obsolete
- def generate_deps_for_vhdl_in_modules(self):
- all_files = self.extract_files_from_all_modules(extensions="vhd")
- p.vprint("All vhdl files:")
- for file in all_files:
- p.vprint(str(file) + ':' + file.library)
- for file in all_files:
- file.search_for_package()
- file.search_for_use()
-
- package_file_dict = {}
- for file in all_files:
- packages = file.package #look for package definitions
- if len(packages) != 0: #if there are some packages in the file
- for package in packages:
- if package in package_file_dict:
- p.echo("There might be a problem... Compilation unit " + package +
- " has several instances:\n\t" + str(file) + "\n\t" + str(package_file_dict[package]))
- package_file_dict[package.lower()] = [package_file_dict[package.lower()], file]#///////////////////////////////////////////////////
- package_file_dict[package.lower()] = file #map found package to scanned file
- file_purename = os.path.splitext(file.name)[0]
- if file_purename in package_file_dict and package_file_dict[file_purename.lower()] != file:
- p.echo("There might be a problem... Compilation unit " + file_purename +
- " has several instances:\n\t" + str(file) + "\n\t" + str(package_file_dict[file_purename]))
- package_file_dict[file_purename.lower()] = file
-
- p.vpprint(package_file_dict)
-
- file_file_dict = {}
- for file in all_files:
- for unit in file.use:
- if unit[1].lower() in package_file_dict:
- if unit[0].lower() == package_file_dict[unit[1].lower()].library:
- if file in file_file_dict:
- file_file_dict[file].append(package_file_dict[unit[1].lower()])
- else:
- file_file_dict[file] = [package_file_dict[unit[1].lower()]]
- else:
- p.echo("Cannot resolve dependency: " + str(file) + " depends on "
- +"compilation unit " + str(unit) + ", which cannot be found")
- for file in all_files:
- if file not in file_file_dict:
- file_file_dict[file] = []
- p.vpprint(file_file_dict)
- return file_file_dict
+ return f_set
\ No newline at end of file
diff --git a/synthesis/srcfile.py b/synthesis/srcfile.py
index 4eb1c365..a23d6bd9 100644
--- a/synthesis/srcfile.py
+++ b/synthesis/srcfile.py
@@ -178,6 +178,8 @@ class SourceFileSet(list):
raise RuntimeError("Expected object, not a string")
elif isinstance(files, list):
self.files.extend(files)
+ elif files == None:
+ p.vprint("Got None as a file.\n Ommiting")
else: #single file, not a list
self.files.append(files)
#if(isinstance(files, SourceFileSet)):
@@ -210,6 +212,7 @@ class SourceFileFactory:
extension = tmp[len(tmp)-1]
p.vprint("SFF> " + path);
+ nf = None
if extension == 'vhd' or extension == 'vhdl':
nf = VHDLFile(path, library)
elif extension == 'v' or extension == 'sv':
--
GitLab