From 8257ede7710edd9f9e3d8f41949880bf1d31267b Mon Sep 17 00:00:00 2001
From: Pawel Szostek <pawel.szostek@gmail.com>
Date: Wed, 11 May 2011 14:48:38 +0200
Subject: [PATCH] Add ModulePool and ModuleFetcher classes
---
synthesis/fetch.py | 248 +++++++++++++++++++++++++++-------------
synthesis/fetch_new.py | 101 ----------------
synthesis/global_mod.py | 1 +
synthesis/hdlmake.py | 19 +--
synthesis/module.py | 97 ++--------------
5 files changed, 188 insertions(+), 278 deletions(-)
delete mode 100644 synthesis/fetch_new.py
diff --git a/synthesis/fetch.py b/synthesis/fetch.py
index fc585644..5a8346aa 100644
--- a/synthesis/fetch.py
+++ b/synthesis/fetch.py
@@ -5,98 +5,182 @@ import msg as p
import global_mod
import path
-def fetch_from_svn(url, revision = None, fetchto = None):
- if fetchto == None:
- fetchto = global_mod.fetchto
-
- if not os.path.exists(fetchto):
- os.mkdir(fetchto)
-
- cur_dir = os.getcwd()
- os.chdir(fetchto)
- basename = path.url_basename(url)
-
- cmd = "svn checkout {0} " + basename
- if revision:
- cmd = cmd.format(url + '@' + revision)
- else:
- cmd = cmd.format(url)
-
- p.vprint(cmd)
- os.system(cmd)
- os.chdir(cur_dir)
-
-def fetch_from_git(url, revision = None, fetchto = None):
- if fetchto == None:
- fetchto = global_mod.fetchto
-
- basename = path.url_basename(url)
- if basename.endswith(".git"):
- basename = basename[:-4] #remove trailing .git
-
- if not os.path.exists(fetchto):
- if not global_mod.fetch:
- return None;
- os.mkdir(fetchto)
-
- if os.path.exists(fetchto+"/"+basename):
- if global_mod.options.fetch:
- update_only = True;
- do_fetch = True;
- else:
- return True;
- else:
- if(global_mod.options.fetch):
- update_only = False;
- do_fetch = True;
- else:
- return None
- rval = True
- if do_fetch:
+class ModuleFetcher:
+
+ def __init__(self):
+ pass
+
+ def fetch_single_module(self, module):
+ involved_modules = []
+ p.vprint("Fetching manifest: " + str(module.manifest))
+
+ if(module.source == "local"):
+ p.vprint("ModPath: " + module.path);
+ module.parse_manifest()
+
+ if module.root_module != None:
+ root_module = module.root_module
+ p.vprint("Encountered root manifest: " + str(root_module))
+ new_modules = self.fetch_recursively(root_module)
+ involved_modules.extend(new_modules)
+
+ for i in module.local:
+ p.vprint("Modules waiting in fetch queue:"+
+ ' '.join([str(module.git), str(module.svn), str(module.local)]))
+
+ for module in module.svn:
+ p.vprint("[svn] Fetching to " + module.fetchto)
+ path = self.__fetch_from_svn(module)
+ module.path = path
+ module.source = "local"
+ module.isparsed = False
+ p.vprint("[svn] Local path " + module.path)
+ involved_modules.append(module)
+
+ for module in module.git:
+ p.vprint("[git] Fetching to " + module.fetchto)
+ path = self.__fetch_from_git(module)
+ module.path = path
+ module.source = "local"
+ module.isparsed = False
+ module.manifest = module.search_for_manifest();
+ p.vprint("[git] Local path " + module.path);
+ involved_modules.append(module)
+
+ for module in modle.local:
+ involved_modules.append(module)
+
+ return involved_modules
+
+ def __fetch_from_svn(self, module):
+ fetchto = module.fetchto
+ if not os.path.exists(fetchto):
+ os.mkdir(fetchto)
cur_dir = os.getcwd()
os.chdir(fetchto)
+ url, rev = __parse_repo_url(module.url)
- if update_only:
- fdir = fetchto+"/"+basename;
- os.chdir(fdir);
- cmd = "git pull"
- p.vprint(cmd);
- if os.system(cmd) != 0:
- rval = False
- os.chdir(fetchto)
+ basename = path.url_basename(url)
- else:
- cmd = "git clone " + url
- p.vprint(cmd);
- if os.system(cmd) != 0:
- rval = False
-
+ cmd = "svn checkout {0} " + basename
+ if rev:
+ cmd = cmd.format(url + '@' + rev)
+ else:
+ cmd = cmd.format(url)
- if revision and rval:
- os.chdir(basename)
- if os.system("git checkout " + revision) != 0:
- rval = False
-
+ rval = True
+
+ p.vprint(cmd)
+ if os.system(cmd) != 0:
+ rval = False
os.chdir(cur_dir)
- return rval
+ module.isfetched = True
+ module.path = os.path.join(fetchto, basename)
+ return rval
+
+ def __fetch_from_git(self, module):
+ fetchto = module.fetchto
+ if not os.path.exists(fetchto):
+ os.mkdir(fetchto)
+
+ cur_dir = os.getcwd()
+ os.chdir(fetchto)
+ url, rev = __parse_repo_url(module.url)
+
+ basename = path.url_basename(url)
+
+ if basename.endswith(".git"):
+ basename = basename[:-4] #remove trailing .git
+
+ if not os.path.exists(os.path.join(fetchto, basename)):
+ update_only = False
+ else:
+ update_only = True
+
+ if update_only:
+ cmd = "git --git-dir="+basename+"/.git pull"
+ else:
+ cmd = "git clone " + url
-def parse_repo_url(url) :
- """
- Check if link to a repo seems to be correct
- """
- import re
- url_pat = re.compile("[ \t]*([^ \t]+)[ \t]*(@[ \t]*(.+))?[ \t]*")
- url_match = re.match(url_pat, url)
+ rval = True
- if url_match == None:
- p.echo("Not a correct repo url: {0}. Skipping".format(url))
- if url_match.group(3) != None: #there is a revision given
- ret = (url_match.group(1), url_match.group(3))
- else:
- ret = url_match.group(1)
- return ret
+ p.vprint(cmd)
+ if os.system(cmd) != 0:
+ rval = False
+ if rev and rval:
+ os.chdir(basename)
+ if os.system("git checkout " + revision) != 0:
+ rval = False
+ os.chdir(cur_dir)
+ module.isfetched = True
+ module.path = os.path.join(fetchto, basename)
+ return rval
+
+ def __parse_repo_url(self, url) :
+ """
+ Check if link to a repo seems to be correct. Filter revision number
+ """
+ import re
+ url_pat = re.compile("[ \t]*([^ \t]+)[ \t]*(@[ \t]*(.+))?[ \t]*")
+ url_match = re.match(url_pat, url)
+
+ if url_match == None:
+ p.echo("Not a correct repo url: {0}. Skipping".format(url))
+ if url_match.group(3) != None: #there is a revision given
+ ret = (url_match.group(1), url_match.group(3))
+ else:
+ ret = (url_match.group(1), None)
+
+class ModulePool:
+ def __init__(self, top_module):
+ self.top_module = top_module
+ self.modules = []
+
+ def __iter__(self):
+ return self.modules.__iter__()
+
+ def __len__(self):
+ return len(self.modules)
+
+ def __contains__(self,v):
+ return v in self.files
+
+ def __getitem__(self,v):
+ return self.files(v)
+
+ def __str__(self):
+ return str([str(m) for m in self.modules])
+
+ def add(self, module):
+ if not isinstance(module, Module):
+ raise RuntimeError("Expecting a Module instance")
+ for mod in self.modules:
+ if mod.url == module.url:
+ return False
+ self.modules.append(module)
+ return True
+
+ def fetch_all(self):
+ fetcher = ModuleFetcher()
+ fetch_queue = [self.top_module]
+
+ while len(fetch_queue) > 0:
+ cur_mod = fetch_queue.pop()
+ new_modules = fetcher.fetch_single_module(cur_mod)
+ for mod in new_modules:
+ ret = self.add(mod)
+ if ret == True:
+ fetch_queue.append(mod)
+ else:
+ pass
+
+ def is_everything_fetched(self):
+ for mod in self.modules:
+ if mod.is_fetched_recursively() == False:
+ return False
+ return True
diff --git a/synthesis/fetch_new.py b/synthesis/fetch_new.py
deleted file mode 100644
index d6c556de..00000000
--- a/synthesis/fetch_new.py
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-import os
-import msg as p
-import global_mod
-import path
-
-
-class ModuleFetcher:
-
- def __init__(self, fetch_dir = None):
- pass
-
-# inputs:
-# - type of the module (local/git/svn)
- def fetch(self, mod_type, mod_url):
- pass
-
- def
-
-class ModulePool:
-
-
-
-
-def fetch_from_svn(url, revision = None, fetchto = None):
- if fetchto == None:
- fetchto = global_mod.fetchto
-
- if not os.path.exists(fetchto):
- os.mkdir(fetchto)
-
- cur_dir = os.getcwd()
- os.chdir(fetchto)
- basename = path.url_basename(url)
-
- cmd = "svn checkout {0} " + basename
- if revision:
- cmd = cmd.format(url + '@' + revision)
- else:
- cmd = cmd.format(url)
-
- p.vprint(cmd)
- os.system(cmd)
- os.chdir(cur_dir)
-
-def fetch_from_git(url, revision = None, fetchto = None):
- if fetchto == None:
- fetchto = global_mod.fetchto
-
- basename = path.url_basename(url)
- if basename.endswith(".git"):
- basename = basename[:-4] #remove trailing .git
-
- if not os.path.exists(fetchto):
- os.mkdir(fetchto)
-
- if not os.path.exists(fetchto+"/"+basename):
- update_only = False
- else:
- update_only = True
-
- cur_dir = os.getcwd()
- os.chdir(fetchto)
-
-
- if update_only:
- cmd = "git --git-dir="+basename+"/.git pull"
- else:
- cmd = "git clone " + url
-
- rval = True
- if os.system(cmd) != 0:
- rval = False
-
- if revision and rval:
- os.chdir(basename)
- if os.system("git checkout " + revision) != 0:
- rval = False
-
- os.chdir(cur_dir)
- return rval
-
-
-def parse_repo_url(url) :
- """
- Check if link to a repo seems to be correct
- """
- import re
- url_pat = re.compile("[ \t]*([^ \t]+)[ \t]*(@[ \t]*(.+))?[ \t]*")
- url_match = re.match(url_pat, url)
-
- if url_match == None:
- p.echo("Not a correct repo url: {0}. Skipping".format(url))
- if url_match.group(3) != None: #there is a revision given
- ret = (url_match.group(1), url_match.group(3))
- else:
- ret = url_match.group(1)
- return ret
-
-
diff --git a/synthesis/global_mod.py b/synthesis/global_mod.py
index 00c35f26..62b80b7e 100644
--- a/synthesis/global_mod.py
+++ b/synthesis/global_mod.py
@@ -6,6 +6,7 @@ t0 = None
ssh = None
top_module = None
global_target = "''"
+modules_pool = None
cwd = None
opt_map = None
hdlm_path = "hdl_make"
diff --git a/synthesis/hdlmake.py b/synthesis/hdlmake.py
index 2ed174d6..5f6a9a68 100755
--- a/synthesis/hdlmake.py
+++ b/synthesis/hdlmake.py
@@ -16,7 +16,7 @@ import msg as p
import optparse
from module import Module
from helper_classes import Manifest, ManifestParser
-from fetch import *
+from fetch import ModulePool
def main():
@@ -79,14 +79,17 @@ def main():
global_mod.top_module.parse_manifest()
global_mod.global_target = global_mod.top_module.target
- global_mod.top_module.fetch()
+ #global_mod.top_module.fetch()
else:
p.echo("No manifest found. At least an empty one is needed")
quit()
+ global_mod.modules_pool = ModulePool(global_mod.top_module)
global_mod.ssh = Connection(options.synth_user, options.synth_server)
- if global_mod.options.local == True:
+ if global_mod.options.fetch == True:
+ fetch()
+ elif global_mod.options.local == True:
local_synthesis()
elif global_mod.options.remote == True:
remote_synthesis()
@@ -106,9 +109,9 @@ def generate_pseudo_ipcore():
os.system("make -f Makefile.ipcore")
def fetch():
- modules = global_mod.top_module.fetch()
- p.vprint("Involved modules:")
- p.vprint([str(m) for m in modules])
+ pool = global_mod.modules_pool
+ pool.fetch_all()
+ p.vprint(str(pool))
def generate_fetch_makefile():
from depend import MakefileWriter
@@ -146,11 +149,11 @@ def generate_makefile():
def generate_ise_makefile(top_mod):
from depend import MakefileWriter
make_writer = MakefileWriter()
- make_writer.
+ make_writer.generate_ise_makefile()
def generate_ise_project(fileset, top_mod):
from flow import ISEProject, ISEProjectProperty
-
+
prj = ISEProject()
prj.add_files(fileset.files)
prj.add_libs(fileset.get_libs())
diff --git a/synthesis/module.py b/synthesis/module.py
index f2ee8a73..96fdce49 100644
--- a/synthesis/module.py
+++ b/synthesis/module.py
@@ -49,6 +49,7 @@ class Module(object):
self.options["manifest"] = manifest
if source == "local":
+ self.path = self.url
self.options["isfetched"] = True
else:
self.options["isfetched"] = isfetched
@@ -63,7 +64,7 @@ class Module(object):
if fetchto != None:
self.options["fetchto"] = fetchto
else:
- self.options["fetchto"] = parent
+ self.options["fetchto"] = parent.fetchto
self.options["isparsed"] = False
basename = path_mod.url_basename(self.options["url"])
@@ -246,91 +247,13 @@ class Module(object):
self.isparsed = True
- def is_fetched(self):
- return self.isfetched
-
- def fetch(self):
-
- if self.source == "local":
- self.path = self.url
-
- involved_modules = [self]
- modules_queue = [self]
-
- p.vprint("Fetching manifest: " + str(self.manifest))
-
- while len(modules_queue) > 0:
- if(self.source == "local"):
- cur_mod = modules_queue.pop()
- p.vprint("ModPath: " + cur_mod.path);
- cur_mod.parse_manifest()
-
- if cur_mod.root_module != None:
- root_module = cur_mod.root_module
- p.vprint("Encountered root manifest: " + str(root_module))
- new_modules = root_module.fetch()
- involved_modules.extend(new_modules)
- modules_queue.extend(new_modules)
-
- for i in cur_mod.local:
- p.vprint("Modules waiting in fetch queue:"+
- ' '.join([str(cur_mod.git), str(cur_mod.svn), str(cur_mod.local)]))
-
- for module in cur_mod.svn:
- p.vprint("Fetching to " + module.fetchto)
- path = module.__fetch_from_svn()
- module.path = path
- module.source = "local"
- module.isparsed = False;
- involved_modules.append(module)
- modules_queue.append(module)
-
- for module in cur_mod.git:
- p.vprint("[git] Fetching to " + module.fetchto)
- path = module.__fetch_from_git()
- module.path = path
- module.source = "local"
- module.isparsed = False;
- module.manifest = module.search_for_manifest();
- p.vprint("[git] Local path " + module.path);
- involved_modules.append(module)
- modules_queue.append(module)
-
- for module in cur_mod.local:
- involved_modules.append(module)
- modules_queue.append(module)
-
- p.vprint("Modules scan queue: " + str(modules_queue))
-
- p.vprint("All found manifests have been scanned")
- return involved_modules
-
- def __fetch_from_svn(self):
- fetchto = self.fetchto
- if not os.path.exists(fetchto):
- os.mkdir(fetchto)
-
- cur_dir = os.getcwd()
- os.chdir(fetchto)
- p.echo(os.getcwd())
- basename = path_mod.url_basename(self.url)
-
- cmd = "svn checkout {0} {1}"
- cmd = cmd.format(self.url, basename)
-
- p.vprint(cmd)
- os.system(cmd)
- os.chdir(cur_dir)
- self.isfetched = True
- self.path = os.path.join(fetchto, basename)
-
- return os.path.join(fetchto, basename)
-
- def __fetch_from_git(self):
-# p.vprint(self.fetchto);
- basename = path_mod.url_basename(self.url)
- self.isfetched = fetch_from_git(self.url, None, self.fetchto);
- return os.path.join(self.fetchto, basename)
+ def is_fetched_recursively(self):
+ if not self.isfetched:
+ return False
+ for mod in self.local + self.svn + self.git:
+ if mod.is_fetched_recursively() == False:
+ return False
+ return True
def make_list_of_modules(self):
p.vprint("Making list of modules for " + str(self))
@@ -384,7 +307,7 @@ class Module(object):
f_set.add(m.fileset);
return f_set
-
+#obsolete
def generate_deps_for_vhdl_in_modules(self):
all_files = self.extract_files_from_all_modules(extensions="vhd")
p.vprint("All vhdl files:")
--
GitLab