Skip to content

H
Hdlmake
Commit dda33958 authored by Paweł Szostek's avatar Paweł Szostek
Browse files
Options

implement git submodules support

parent c82de3ff
Hide whitespace changes
Inline Side-by-side
Showing with 194 additions and 75 deletions
hdlmake/__main__.py
View file @ dda33958
......@@ -30,6 +30,7 @@ from util.termcolor import colored
from manifest_parser import ManifestParser
from module_pool import ModulePool
from env import Env
import fetch as fetch_mod
from action import (CheckCondition, CleanModules, FetchModules, GenerateFetchMakefile,
GenerateISEMakefile, GenerateISEProject, ListFiles,
ListModules, MergeCores, GenerateQuartusProject,
......@@ -96,7 +97,9 @@ def main():
options = condition_check.parse_args(sys.argv[2:])
env = Env(options)
env.check_env()
CheckCondition(modules_pool=None, options=options, env=env).run()
CheckCondition(modules_pool=None,
options=options,
env=env).run()
quit()
else:
options = parser.parse_args(sys.argv[1:])
......@@ -110,8 +113,11 @@ def main():
logging.debug(str(options))
modules_pool = ModulePool()
modules_pool.new_module(parent=None, url=os.getcwd(), source="local",
fetchto=".", process_manifest=False)
modules_pool.new_module(parent=None,
url=os.getcwd(),
source=fetch_mod.LOCAL,
fetchto=".",
process_manifest=False)
# Setting top_module as top module of design (ModulePool class)
if modules_pool.get_top_module().manifest is None:
......@@ -147,16 +153,25 @@ def main():
if options.command == "auto":
logging.info("Running automatic flow.")
if not top_mod.action:
logging.error("`action' manifest variable has to be specified.\n"
logging.error("`action' manifest variable has to be specified. "
"Otherwise hdlmake doesn't know how to handle the project")
quit()
if top_mod.action == "simulation":
sim = GenerateSimulationMakefile(modules_pool=modules_pool, options=options, env=env)
sim = GenerateSimulationMakefile(modules_pool=modules_pool,
options=options,
env=env)
sim.run()
quit()
elif top_mod.action == "synthesis":
syn = GenerateISEMakefile(modules_pool=modules_pool, options=options, env=env)
ise = GenerateISEProject(modules_pool=modules_pool, options=options, env=env)
remote = GenerateRemoteSynthesisMakefile(modules_pool=modules_pool, options=options, env=env)
syn = GenerateISEMakefile(modules_pool=modules_pool,
options=options,
env=env)
ise = GenerateISEProject(modules_pool=modules_pool,
options=options,
env=env)
remote = GenerateRemoteSynthesisMakefile(modules_pool=modules_pool,
options=options,
env=env)
syn.run()
ise.run()
remote.run()
......@@ -188,7 +203,9 @@ def main():
elif options.command == "quartus-project":
action = GenerateQuartusProject
action_instance = action(modules_pool=modules_pool, options=options, env=env)
action_instance = action(modules_pool=modules_pool,
options=options,
env=env)
try:
action_instance.run()
......
hdlmake/action/ise_makefile.py
View file @ dda33958
......@@ -23,10 +23,12 @@ from __future__ import print_function
from action import Action
import logging
import global_mod
import dep_solver
class GenerateISEMakefile(Action):
def _check_manifest(self):
self._check_manifest_variable_is_set("syn_tool")
def run(self):
logging.info("Generating makefile for local synthesis.")
......
hdlmake/action/ise_project.py
View file @ dda33958
......@@ -37,7 +37,6 @@ class GenerateISEProject(Action):
def _check_manifest(self):
self._check_manifest_variable_is_set("top_module")
self._check_manifest_variable_is_set("syn_device")
self._check_manifest_variable_is_set("syn_device")
self._check_manifest_variable_is_set("syn_grade")
self._check_manifest_variable_is_set("syn_package")
......@@ -84,6 +83,7 @@ class GenerateISEProject(Action):
prj.load_xml(top_mod.syn_project)
else:
prj.add_initial_properties()
logging.info("Writing down .xise project file")
prj.emit_xml(top_mod.syn_project)
def _write_project_vhd(self):
......
hdlmake/action/list_modules.py
View file @ dda33958
......@@ -21,6 +21,7 @@
from action import Action
from util import path
import fetch
class ListModules(Action):
......@@ -32,7 +33,7 @@ class ListModules(Action):
print(m.url+'\n')
else:
print(path.relpath(m.path))
if m.source in ["svn", "git"]:
if m.source in [fetch.SVN, fetch.GIT]:
print("# "+m.url)
if m.parent:
print("# defined in %s" % m.parent.url)
......
hdlmake/fetch/__init__.py
View file @ dda33958
......@@ -22,3 +22,8 @@
from backend_factory import BackendFactory
from svn import Svn
from git import Git
GIT = 1
SVN = 2
LOCAL = 3
GITSUBMODULE = 4
\ No newline at end of file
hdlmake/fetch/backend_factory.py
View file @ dda33958
......@@ -19,9 +19,10 @@
# You should have received a copy of the GNU General Public License
# along with Hdlmake. If not, see <http://www.gnu.org/licenses/>.
from git import Git
from git import (Git, GitSubmodule)
from svn import Svn
import logging
import fetch
class Local(object):
......@@ -37,12 +38,14 @@ class BackendFactory(object):
pass
def get_backend(self, module):
if module.source == "local":
if module.source == fetch.LOCAL:
return Local()
else:
logging.info("Investigating module: " + str(module) +
"[parent: " + str(module.parent) + "]")
if module.source == "svn":
if module.source == fetch.SVN:
return Svn()
if module.source == "git":
if module.source == fetch.GIT:
return Git()
if module.source == fetch.GITSUBMODULE:
return GitSubmodule()
hdlmake/fetch/git.py
View file @ dda33958
......@@ -24,13 +24,47 @@ from util import path
import logging
from tempfile import TemporaryFile
from subprocess import Popen, PIPE
import fetch
class GitSubmodule(object):
def fetch(self, module):
if module.source != fetch.GITSUBMODULE:
raise ValueError("This backend should get git modules only.")
cur_dir = os.getcwd()
os.chdir(module.fetchto)
os.system("git submodule init")
os.system("git submodule update")
os.chdir(cur_dir)
class Git(object):
def __init__(self):
pass
@staticmethod
def get_git_submodules(module):
submodule_dir = path.rel2abs(module.path)
logging.info("Checking git submodules in %s" % submodule_dir)
cmd = "(cd %s && git config --list | grep submodule | sed 's/.*=//')" % submodule_dir
config_submodules = Popen(cmd,
stdout=PIPE,
stdin=PIPE,
shell=True)
config_submodules = [line.strip() for line in config_submodules.stdout.readlines()]
cmd = "(cd %s && cat ./.gitmodules | grep url | sed 's/url = //')" % submodule_dir
dotgitmodules_submodules = Popen(cmd,
stdout=PIPE,
stdin=PIPE,
shell=True)
dotgitmodules_submodules = [line.strip() for line in dotgitmodules_submodules.stdout.readlines()]
set(config_submodules).update(set(dotgitmodules_submodules))
submodules = list(config_submodules)
return submodules
def fetch(self, module):
if module.source != fetch.GIT:
raise ValueError("This backend should get git modules only.")
if not os.path.exists(module.fetchto):
os.mkdir(module.fetchto)
......@@ -41,6 +75,8 @@ class Git(object):
basename = path.url_basename(module.url)
mod_path = os.path.join(module.fetchto, basename)
logging.info("Fetching git module: %s" % mod_path)
if basename.endswith(".git"):
basename = basename[:-4] # remove trailing .git
......@@ -64,12 +100,6 @@ class Git(object):
if os.system(cmd) != 0:
success = False
if success is True:
os.chdir(mod_path)
os.system("git submodule init")
os.system("git submodule update")
os.chdir(cur_dir)
if module.revision is not None and success is True:
logging.debug("cd %s" % mod_path)
os.chdir(mod_path)
......@@ -91,7 +121,12 @@ class Git(object):
try:
os.chdir(path)
git_cmd = 'git log -1 --format="%H" | cut -c1-32'
git_out = Popen(git_cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=stderr, close_fds=True)
git_out = Popen(git_cmd,
shell=True,
stdin=PIPE,
stdout=PIPE,
stderr=stderr,
close_fds=True)
errmsg = stderr.readlines()
if errmsg:
logging.debug("git error message (in %s): %s" % (path, '\n'.join(errmsg)))
......
hdlmake/makefile_writer.py
View file @ dda33958
......@@ -272,12 +272,12 @@ mrproper:
self.initialize()
self.write("#target for fetching all modules stored in repositories\n")
self.write("fetch: ")
self.write(' \\\n'.join(["__"+m.basename+"_fetch" for m in modules_pool if m.source in ["svn", "git"]]))
self.write(' \\\n'.join(["__"+m.basename+"_fetch" for m in modules_pool if m.source in (SVN, GIT)]))
self.write("\n\n")
for module in modules_pool:
basename = module.basename
if module.source == "svn":
if module.source == SVN:
self.write("__"+basename+"_fetch:\n")
self.write("\t\tmkdir -p %s\n" % rp(module.fetchto))
self.write("\t\tPWD=$(shell pwd) ")
......@@ -290,7 +290,7 @@ mrproper:
self.write(c)
self.write("; cd $(PWD) \n\n")
elif module.source == "git":
elif module.source == GIT:
self.write("__"+basename+"_fetch:\n")
self.write("\t\tmkdir -p %s\n" % rp(module.fetchto))
self.write("\t\t")
......
hdlmake/manifest_parser.py
View file @ dda33958
......@@ -52,6 +52,7 @@ class ManifestParser(ConfigParser):
self.add_delimiter()
self.add_option('syn_name', default=None, help="Name of the folder at remote synthesis machine", type='')
self.add_option('syn_tool', default=None, help="Tool to be used in the synthesis", type='')
self.add_option('syn_device', default=None, help="Target FPGA device", type='')
self.add_option('syn_grade', default=None, help="Speed grade of target FPGA", type='')
self.add_option('syn_package', default=None, help="Package variant of target FPGA", type='')
......
hdlmake/module.py
View file @ dda33958
......@@ -36,7 +36,7 @@ class Module(object):
@source.setter
def source(self, value):
if value not in ["svn", "git", "local"]:
if value not in [fetch.GIT, fetch.SVN, fetch.LOCAL, fetch.GITSUBMODULE]:
raise ValueError("Inproper source: " + value)
self._source = value
......@@ -47,7 +47,7 @@ class Module(object):
@property
def basename(self):
from util import path
if self.source == "svn":
if self.source == fetch.SVN:
return path.svn_basename(self.url)
else:
return path.url_basename(self.url)
......@@ -86,6 +86,7 @@ class Module(object):
self.syn_package = None
self.syn_project = None
self.syn_top = None
self.syn_tool = None
self.syn_ise_version = None
self.syn_pre_script = None
self.syn_post_script = None
......@@ -96,17 +97,17 @@ class Module(object):
self.commit_id = None
self.raw_url = url
if source != "local":
if source != fetch.LOCAL:
self.url, self.branch, self.revision = path.url_parse(url)
else:
self.url, self.branch, self.revision = url, None, None
if source == "local" and not os.path.exists(url):
if source == fetch.LOCAL and not os.path.exists(url):
logging.error("Path to the local module doesn't exist:\n" + url
+ "\nThis module was instantiated in: " + str(parent))
quit()
if source == "local":
if source == fetch.LOCAL:
self.path = url
self.isfetched = True
else:
......@@ -117,10 +118,9 @@ class Module(object):
self.path = None
self.isfetched = False
if self.path is not None:
self.manifest = self._search_for_manifest()
else:
self.manifest = None
self.manifest = None
self.git_submodules = []
def __str__(self):
return self.raw_url
......@@ -136,7 +136,7 @@ class Module(object):
else:
return x
return __nonull(self.local) + __nonull(self.git) + __nonull(self.svn)
return __nonull(self.local) + __nonull(self.git) + __nonull(self.svn) + __nonull(self.git_submodules)
def _search_for_manifest(self):
"""
......@@ -183,7 +183,6 @@ class Module(object):
def parse_manifest(self):
if self.manifest_dict:
return
logging.debug(self.path)
if self.isparsed is True or self.isfetched is False:
return
if self.manifest is None:
......@@ -198,6 +197,7 @@ class Module(object):
if self.manifest is None:
logging.debug("No manifest found in module "+str(self))
else:
logging.debug("Parse manifest in: %s" % self.path)
manifest_parser.add_manifest(self.manifest)
if self.parent is None:
......@@ -250,7 +250,10 @@ class Module(object):
"(" + self.path + ")")
quit()
path = path_mod.rel2abs(path, self.path)
local_mods.append(self.pool.new_module(parent=self, url=path, source="local", fetchto=fetchto))
local_mods.append(self.pool.new_module(parent=self,
url=path,
source=fetch.LOCAL,
fetchto=fetchto))
self.local = local_mods
else:
self.local = []
......@@ -310,7 +313,10 @@ class Module(object):
if self.manifest_dict["files"] == []:
self.files = SourceFileSet()
logging.debug("No files in the manifest %s" % self.manifest.path)
try:
logging.debug("No files in the manifest %s" % self.manifest.path)
except AttributeError:
pass
else:
self.manifest_dict["files"] = self._flatten_list(self.manifest_dict["files"])
logging.debug(self.path + str(self.manifest_dict["files"]))
......@@ -343,7 +349,10 @@ class Module(object):
self.manifest_dict["modules"]["svn"] = self._flatten_list(self.manifest_dict["modules"]["svn"])
svn_mods = []
for url in self.manifest_dict["modules"]["svn"]:
svn_mods.append(self.pool.new_module(parent=self, url=url, source="svn", fetchto=fetchto))
svn_mods.append(self.pool.new_module(parent=self,
url=url,
source=fetch.SVN,
fetchto=fetchto))
self.svn = svn_mods
else:
self.svn = []
......@@ -352,11 +361,20 @@ class Module(object):
self.manifest_dict["modules"]["git"] = self._flatten_list(self.manifest_dict["modules"]["git"])
git_mods = []
for url in self.manifest_dict["modules"]["git"]:
git_mods.append(self.pool.new_module(parent=self, url=url, source="git", fetchto=fetchto))
git_mods.append(self.pool.new_module(parent=self,
url=url,
source=fetch.GIT,
fetchto=fetchto))
self.git = git_mods
else:
self.git = []
git_submodule_urls = fetch.Git.get_git_submodules(self)
for submodule_url in git_submodule_urls:
self.git_submodules.append(self.pool.new_module(parent=self,
url=submodule_url,
fetchto=self.path,
source=fetch.GITSUBMODULE))
self.target = self.manifest_dict["target"].lower()
self.action = self.manifest_dict["action"].lower()
......@@ -364,6 +382,7 @@ class Module(object):
self.syn_name = self.manifest_dict["syn_project"][:-5] # cut out .xise from the end
else:
self.syn_name = self.manifest_dict["syn_name"]
self.syn_tool = self.manifest_dict["syn_tool"]
self.syn_device = self.manifest_dict["syn_device"]
self.syn_grade = self.manifest_dict["syn_grade"]
self.syn_package = self.manifest_dict["syn_package"]
......@@ -385,14 +404,13 @@ class Module(object):
else:
self.revision = revision
else:
if self.source == "svn":
if self.source == fetch.SVN:
self.revision = fetch.Svn.check_revision_number(self.path)
elif self.source == "git":
elif self.source == fetch.GIT:
self.revision = fetch.Git.check_commit_id(self.path)
def _make_list_of_paths(self, list_of_paths):
paths = []
logging.debug(str(list_of_paths))
for filepath in list_of_paths:
if self._check_filepath(filepath):
paths.append(path_mod.rel2abs(filepath, self.path))
......
hdlmake/module_pool.py
View file @ dda33958
......@@ -25,9 +25,9 @@ import os
import logging
import global_mod
import new_dep_solver as dep_solver
from util import path
import sys
from util import path as path_mod
from fetch import BackendFactory
import fetch
from subprocess import PIPE, Popen
......@@ -38,13 +38,14 @@ class ModulePool(list):
self.global_fetch = os.getenv("HDLMAKE_COREDIR")
def get_module_by_path(self, path):
path = path_mod.rel2abs(path)
for module in self:
if module.path == path:
return module
return None
def get_fetchable_modules(self):
return [m for m in self if m.source != "local"]
return [m for m in self if m.source != fetch.LOCAL]
def __str__(self):
return str([str(m) for m in self])
......@@ -55,26 +56,11 @@ class ModulePool(list):
return True
return False
def _fetch(self, module):
new_modules = []
logging.debug("Fetching module: " + str(module))
bf = BackendFactory()
fetcher = bf.get_backend(module)
fetcher.fetch(module)
module.parse_manifest()
module.process_manifest()
new_modules.extend(module.local)
new_modules.extend(module.svn)
new_modules.extend(module.git)
return new_modules
def new_module(self, parent, url, source, fetchto, process_manifest=True):
from module import Module
if source != "local":
clean_url, branch, revision = path.url_parse(url)
if source != fetch.LOCAL:
clean_url, branch, revision = path_mod.url_parse(url)
else:
clean_url, branch, revision = url, None, None
if url in [m.raw_url for m in self]: # check if module is not already in the pool
......@@ -97,7 +83,10 @@ class ModulePool(list):
if self.global_fetch: # if there is global fetch parameter (HDLMAKE_COREDIR env variable)
fetchto = self.global_fetch # screw module's particular fetchto
new_module = Module(parent=parent, url=url, source=source, fetchto=fetchto, pool=self)
new_module = Module(parent=parent,
url=url, source=source,
fetchto=fetchto,
pool=self)
self._add(new_module)
if not self.top_module:
global_mod.top_module = new_module
......@@ -118,7 +107,10 @@ class ModulePool(list):
try:
os.chdir(path)
git_out = Popen("git config --get remote.origin.url", stdout=PIPE, shell=True, close_fds=True)
url = git_out.stdout.readlines()[0].strip()
lines = git_out.stdout.readlines()
if len(lines) == 0:
return None
url = lines[0].strip()
if not url: # try svn
svn_out = Popen("svn info | grep 'Repository Root' | awk '{print $NF}'", stdout=PIPE, shell=True, close_fds=True)
url = svn_out.stdout.readlines()[0].strip()
......@@ -143,6 +135,23 @@ class ModulePool(list):
self.append(new_module)
return True
def _fetch(self, module):
new_modules = []
logging.debug("Fetching module: " + str(module))
bf = BackendFactory()
fetcher = bf.get_backend(module)
fetcher.fetch(module)
module.parse_manifest()
module.process_manifest()
new_modules.extend(module.local)
new_modules.extend(module.svn)
new_modules.extend(module.git)
new_modules.extend(module.git_submodules)
return new_modules
def fetch_all(self, unfetched_only=False, flatten=False):
fetch_queue = [m for m in self]
......@@ -178,7 +187,6 @@ class ModulePool(list):
files = self.build_global_file_list()
assert isinstance(files, SourceFileSet)
dep_solver.solve(files)
assert isinstance(files, list)
ret = []
for file in files:
try:
......
hdlmake/new_dep_solver.py
View file @ dda33958
......@@ -109,8 +109,39 @@ def solve(fileset):
def make_dependency_sorted_list(fileset, purge_unused=True):
pass
# return list of files sorted in dependency order
# CYCLE_THRESHOLD = 30
# ret = list(fileset)
# cur_idx = 0
# other_file_idx = cur_idx + 1
# swapped = 0
# while True:
# if swapped >= CYCLE_THRESHOLD:
# cur_idx += 1
# if cur_idx >= len(ret):
# break
# if other_file_idx >= len(ret):
# cur_idx += 1
# other_file_idx = cur_idx + 1
# continue
# dep_file = ret[cur_idx]
# other_file = ret[other_file_idx]
# if other_file in dep_file.depends_on:
# ret[cur_idx], ret[other_file_idx] = ret[other_file_idx], ret[cur_idx]
# other_file_idx = cur_idx + 1
# swapped += 1
# else:
# other_file_idx += 1
# return ret
def compare_dep_files(f1, f2):
if f2 in f1.depends_on:
return 1
if f1 in f2.depends_on:
return -1
return 0
ret = list(fileset)
ret = sorted(ret, cmp=compare_dep_files)
return ret
if __name__ == "__main__":
from dep_file import (DepFile)
......
hdlmake/srcfile.py
View file @ dda33958
......@@ -286,7 +286,7 @@ class WBGenFile(File):
pass
class SourceFileSet(list):
class SourceFileSet(set):
def __init__(self):
super(SourceFileSet, self).__init__()
self = []
......@@ -302,11 +302,9 @@ class SourceFileSet(list):
else:
try:
for f in files:
if f not in self:
self.append(f)
super(SourceFileSet, self).add(f)
except: # single file, not a list
if files not in self:
self.append(files)
super(SourceFileSet, self).add(files)
def filter(self, type):
out = SourceFileSet()
......@@ -324,7 +322,7 @@ class SourceFileSet(list):
def get_libs(self):
ret = set()
for file in self.modules_pool.build_global_file_list():
for file in self:
try:
ret.add(file.library)
except:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment