More refactoring, removing redundant info from Module

58884d63 · Javier D. Garcia-Lasheras · 36571faf · 58884d63 · 58884d63 · 58884d63
Commit 58884d63 authored Jul 27, 2016 by Javier D. Garcia-Lasheras
12 changed files
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -547,7 +547,6 @@ And for the Verilog synthesis top Manifest.py:
 .. code-block:: python
-   target = "xilinx"
   action = "synthesis"
   syn_device = "xc6slx45t"
@@ -563,7 +562,6 @@ And for the Verilog synthesis top Manifest.py:
 We can see that the only difference is that each of the top synthesis Manifest.py points to its specific Verilog/VHDL top module describing the interface for the constrained FPGA design. The other Manifest.py variables are common for both languages and they means:
- ``target``: specific targeted FPGA architecture
 - ``action``: indicates that this is a synthesis process
 - ``syn_device``: indicates the specific FPGA device
 - ``syn_family``: indicates the specific FPGA family
@@ -750,6 +748,8 @@ As a very simple example, we can introduce both extra commands in the top synthe
   }
+.. note:: the ``target`` parameter is used as a condition code variable in this specific example
 **Simulation:**
 Now, if we want to add external commands to a simulation top makefile, the following parameters must be introduced:
@@ -1207,8 +1207,6 @@ Basic synthesis variables:
 +-----------------+-------------+-----------------------------------------------------------------+-----------+
 | Name            | Type        | Description                                                     | Default   |
 +=================+=============+=================================================================+===========+
-| target          | str         | What is the target architecture                                 | ""        |
-+-----------------+-------------+-----------------------------------------------------------------+-----------+
 | syn_top         | str         | Top level module for synthesis                                  | None      |
 +-----------------+-------------+-----------------------------------------------------------------+-----------+
 | syn_tool        | str         | Tool to be used in the synthesis                                | None      |

--- a/hdlmake/fetch/__init__.py
+++ b/hdlmake/fetch/__init__.py
@@ -19,6 +19,6 @@
 # You should have received a copy of the GNU General Public License
 # along with Hdlmake.  If not, see <http://www.gnu.org/licenses/>.
-from .constants import (GIT, GITSUBMODULE, SVN, LOCAL, fetch_type_lookup)
+from .constants import (GIT, SVN, LOCAL, fetch_type_lookup)
 from .git import Git
 from .svn import Svn
--- a/hdlmake/fetch/constants.py
+++ b/hdlmake/fetch/constants.py
@@ -25,11 +25,10 @@
 GIT = 1
 SVN = 2
 LOCAL = 3
-GITSUBMODULE = 4
 from .svn import Svn
-from .git import (Git, GitSubmodule)
+from .git import Git
 from .local import Local
 from .backend_factory import BackendFactory
@@ -39,4 +38,3 @@ fetch_type_lookup = BackendFactory()
 fetch_type_lookup.register_backend(GIT, Git)
 fetch_type_lookup.register_backend(SVN, Svn)
 fetch_type_lookup.register_backend(LOCAL, Local)
-fetch_type_lookup.register_backend(GITSUBMODULE, GitSubmodule)
--- a/hdlmake/fetch/git.py
+++ b/hdlmake/fetch/git.py
@@ -25,21 +25,10 @@ import logging
 import platform
 from tempfile import TemporaryFile
 from subprocess import Popen, PIPE
-from .constants import (GIT, GITSUBMODULE)
+from .constants import GIT
 from .fetcher import Fetcher
-class GitSubmodule(Fetcher):
-    def fetch(self, module):
-        if module.source != GITSUBMODULE:
-            raise ValueError("This backend should get git modules only.")
-        cur_dir = module.pool.top_module.path
-        os.chdir(module.fetchto)
-        os.system("git submodule init")
-        os.system("git submodule update")
-        os.chdir(cur_dir)
 class Git(Fetcher):
    def __init__(self):
        pass
@@ -60,70 +49,20 @@ class Git(Fetcher):
        finally:
            os.chdir(cur_dir)
-    @staticmethod
-    def get_git_submodules(module):
-        submodule_dir = path.rel2abs(module.path)
-        logging.debug("Checking git submodules in %s" % submodule_dir)
-        cur_dir = module.pool.top_module.path
-        try:
-            os.chdir(submodule_dir)
-            if not os.path.exists(".gitmodules"):
-                return {}
-            #"git config --list" | grep submodule | sed 's/.*=//')" % submodule_dir
-            config_submodules = {}
-            config_content = Popen("git config -f .gitmodules --list",
-                                      stdout=PIPE,
-                                      stdin=PIPE,
-                                      shell=True)
-            config_lines = [line.strip() for line in config_content.stdout.readlines()]
-            """try to parse sth like this:
-paszoste@oplarra1:~/beco/hdlmake-tests/wr-switch-hdl$ git config -f .gitmodules --list
-submodule.ip_cores/general-cores.path=ip_cores/general-cores
-submodule.ip_cores/general-cores.url=git://ohwr.org/hdl-core-lib/general-cores.git
-submodule.ip_cores/wr-cores.path=ip_cores/wr-cores
-submodule.ip_cores/wr-cores.url=git://ohwr.org/hdl-core-lib/wr-cores.git
-"""
-            config_submodule_lines = [line for line in config_lines if line.startswith("submodule")]
-            for line in config_submodule_lines:
-                line_split = line.split("=")
-                lhs = line_split[0]
-                rhs = line_split[1]
-                lhs_split = lhs.split(".")
-                module_name = '.'.join(lhs_split[1:-1])
-                if module_name not in config_submodules:
-                    config_submodules[module_name] = {}
-                config_submodules[module_name][lhs_split[-1]] = rhs
-            #"(cd %s && cat ./.gitmodules 2>/dev/null | grep url | sed 's/url = //')" % submodule_dir
-            #try:
-            ##    dotgitmodules_file = open(".gitmodules", 'r')
-             #   dotgitmodules_lines = dotgitmodules_file.readlines()
-             #   url_lines = [line for line in dotgitmodules_lines if 'url' in line]
-             #   dotgitmodules_submodules = [line.split(" = ")[-1].strip() for line in url_lines]
-             #  set(config_submodules).update(set(dotgitmodules_submodules))
-            #except IOError:
-             #   pass  # no .gitmodules file
-            if len(list(config_submodules)) > 0:
-                logging.info("Found git submodules in %s: %s" % (module.path, str(config_submodules)))
-        finally:
-            os.chdir(cur_dir)
-        return config_submodules
    def fetch(self, module):
+        fetchto = module.fetchto()
        if module.source != GIT:
            raise ValueError("This backend should get git modules only.")
-        if not os.path.exists(module.fetchto):
+        if not os.path.exists(fetchto):
-            os.mkdir(module.fetchto)
+            os.mkdir(fetchto)
        cur_dir = module.pool.top_module.path
        if module.branch is None:
            module.branch = "master"
        basename = path.url_basename(module.url)
-        mod_path = os.path.join(module.fetchto, basename)
+        mod_path = os.path.join(fetchto, basename)
        logging.info("Fetching git module: %s" % mod_path)
@@ -142,7 +81,7 @@ submodule.ip_cores/wr-cores.url=git://ohwr.org/hdl-core-lib/wr-cores.git
        else:
            logging.info("Cloning module %s" % mod_path)
            cmd = "(cd {0} && git clone -b {2} {1})"
-            cmd = cmd.format(module.fetchto, module.url, module.branch)
+            cmd = cmd.format(fetchto, module.url, module.branch)
        success = True

--- a/hdlmake/fetch/svn.py
+++ b/hdlmake/fetch/svn.py
@@ -33,14 +33,16 @@ class Svn(Fetcher):
        pass
    def fetch(self, module):
-        if not os.path.exists(module.fetchto):
-            os.mkdir(module.fetchto)
+        fetchto = module.fetchto()
+        if not os.path.exists(fetchto):
+            os.mkdir(fetchto)
        cur_dir = module.pool.top_module.path
-        os.chdir(module.fetchto)
+        os.chdir(fetchto)
-        basename = path.url_basename(module.url)
+        basename = path.svn_basename(module.url)
-        mod_path = os.path.join(module.fetchto, basename)
+        mod_path = os.path.join(fetchto, basename)
        cmd = "svn checkout {0} " + module.basename
        if module.revision:
@@ -57,7 +59,7 @@ class Svn(Fetcher):
        os.chdir(cur_dir)
        module.isfetched = True
-        module.path = os.path.join(module.fetchto, module.basename)
+        module.path = os.path.join(fetchto, module.basename)
        return success
    @staticmethod

--- a/hdlmake/manifest_parser.py
+++ b/hdlmake/manifest_parser.py
@@ -111,7 +111,6 @@ class ManifestParser(ConfigParser):
        self.add_delimiter()
        self.add_option('modules', default={}, help="List of local modules", type={})
-        self.add_option('target', default='', help="What is the target architecture", type='')
        self.add_option('action', default='', help="What is the action that should be taken (simulation/synthesis)", type='')
        self.add_allowed_key('modules', key="svn")

--- a/hdlmake/module/__init__.py
+++ b/hdlmake/module/__init__.py
-from .origin import ModuleOrigin
 from .core import ModuleCore
 from .synthesis import ModuleSynthesis
 from .simulation import ModuleSimulation

--- a/hdlmake/module/content.py
+++ b/hdlmake/module/content.py
+import os
 import logging
 from .core import ModuleCore
 from hdlmake import fetch
@@ -5,17 +6,16 @@ from hdlmake.util import path as path_mod
 class ModuleContent(ModuleCore):
    def __init__(self):
+        self.top_entity = None
        # Manifest Files Properties
        self.files = None
        # Manifest Modules Properties
        self.local = []
        self.git = []
        self.svn = []
-        self.git_submodules = []
        super(ModuleContent, self).__init__()
    def process_manifest(self):
-        self._process_manifest_fetch()
        self._process_manifest_files()
        self._process_manifest_modules()
        super(ModuleContent, self).process_manifest()
@@ -44,20 +44,17 @@ class ModuleContent(ModuleCore):
                elif isinstance(f, VHDLFile):
                    f.vcom_opt = self.vcom_opt
-    def _process_manifest_fetch(self):
-        # Fetch configuration
+    def _process_manifest_modules(self):
        if self.manifest_dict["fetchto"] is not None:
            fetchto = path_mod.rel2abs(self.manifest_dict["fetchto"],
                self.path)
-            self.fetchto = fetchto
        else:
-            fetchto = self.fetchto
+            fetchto = self.fetchto()
        self.fetch_pre_cmd = self.manifest_dict["fetch_pre_cmd"]
        self.fetch_post_cmd = self.manifest_dict["fetch_post_cmd"]
-    def _process_manifest_modules(self):
-        fetchto = self.fetchto
        # Process required modules
        if "local" in self.manifest_dict["modules"]:
            local_paths = path_mod.flatten_list(
@@ -103,20 +100,3 @@ class ModuleContent(ModuleCore):
        else:
            self.git = []
-        # Git submodules are temporarly disabled!
-        # -- we need to clearly define the expected behavior
-        # git_submodule_dict = fetch.Git.get_git_submodules(self)
-        # git_toplevel = fetch.Git.get_git_toplevel(self)
-        # for submodule_key in git_submodule_dict.keys():
-        #    url = git_submodule_dict[submodule_key]["url"]
-        #    path = git_submodule_dict[submodule_key]["path"]
-        #    path = os.path.join(git_toplevel, path)
-        #    path = os.path.normpath(path)
-        #    fetchto = os.path.sep.join(path.split(os.path.sep)[:-1])
-        #    self.git_submodules.append(self.pool.new_module(parent=self,
-        #        url=url,
-        #        fetchto=fetchto,
-        #        source=fetch.GITSUBMODULE))
--- a/hdlmake/module/core.py
+++ b/hdlmake/module/core.py
 """Provides the core functionality for the HDLMake module"""
+import os
 import logging
+from hdlmake import fetch
+from hdlmake.util import path as path_mod
-class ModuleCore(object):
+class ModuleConfig(object):
+    """This class containt the base properties and methods that
+    need to be initialized for a proper behavior"""
+    def __init__(self):
+        self.source = None
+        self.parent = None
+        self.url = None
+        self.branch = None
+        self.revision = None
+        self.path = None
+        self.isfetched = False
+    def process_manifest(self):
+        """process_manifest does nothing for ModuleConfig"""
+        pass
+    def basename(self):
+        """Get the basename for the module"""
+        if self.source == fetch.SVN:
+            return path_mod.svn_basename(self.url)
+        else:
+            return path_mod.url_basename(self.url)
+    def fetchto(self):
+        """Get the fetchto folder for the module"""
+        return os.path.dirname(self.path)
+    def init_config(self, parent, url, source, fetchto):
+        """This initializes the module configuration.
+        The function is executed by Module constructor"""
+        self.source = source
+        self.parent = parent
+        if self.source != fetch.LOCAL:
+            self.url, self.branch, self.revision = \
+                path_mod.url_parse(url)
+            basename = self.basename()
+            path = os.path.abspath(os.path.join(fetchto, basename))
+            # Check if the module dir exists and is not empty
+            if os.path.exists(path) and os.listdir(path):
+                self.path = path
+                self.isfetched = True
+                logging.debug("Module %s (parent: %s) is fetched.",
+                    url, self.parent.path)
+            else:
+                self.path = path
+                self.isfetched = False
+                logging.debug("Module %s (parent: %s) is NOT fetched.",
+                    url, self.parent.path)
+        else:
+            self.url, self.branch, self.revision = url, None, None
+            if not os.path.exists(url):
+                logging.error(
+                    "Path to the local module doesn't exist:\n" + url
+                    + "\nThis module was instantiated in: " + str(self.parent))
+                quit()
+            self.path = url
+            self.isfetched = True
+class ModuleCore(ModuleConfig):
    """This is the class providing the module core functionality"""
    def __init__(self):
        # Universal Manifest Properties
        self.library = "work"
-        self.target = None
        self.action = None
        self.force_tool = None
        self.pool = None
        self.top_module = None
        self.manifest_dict = None
-        #super(ModuleCore, self).__init__()
+        self.top_entity = None
+        super(ModuleCore, self).__init__()
    def set_pool(self, pool):
        """Set the associated pool for the module instance"""
@@ -26,7 +93,7 @@ class ModuleCore(object):
        """Method that process the core manifest section"""
        self._process_manifest_force_tool()
        self._process_manifest_universal()
-        #super(ModuleCore, self).process_manifest()
+        super(ModuleCore, self).process_manifest()
    def _process_manifest_force_tool(self):
@@ -46,7 +113,6 @@ class ModuleCore(object):
        #    self.top_module = self.manifest_dict["top_module"]
        # Libraries
        self.library = self.manifest_dict["library"]
-        self.target = self.manifest_dict["target"].lower()
        self.action = self.manifest_dict["action"].lower()
--- a/hdlmake/module/module.py
+++ b/hdlmake/module/module.py
@@ -34,12 +34,11 @@ import logging
 from hdlmake.manifest_parser import ManifestParser
 from hdlmake.util import path as path_mod
-from hdlmake import fetch
+from hdlmake.module import (ModuleSynthesis,
-from hdlmake.module import (ModuleSynthesis, ModuleOrigin,
    ModuleSimulation, ModuleContent, ModuleAltera)
-class Module(ModuleSynthesis, ModuleOrigin,
+class Module(ModuleSynthesis,
    ModuleSimulation, ModuleContent, ModuleAltera):
    """
    This is the class providing the HDLMake module, the basic element
@@ -50,16 +49,13 @@ class Module(ModuleSynthesis, ModuleOrigin,
        """Calculate and initialize the origin attributes: path, source..."""
        assert url is not None
        assert source is not None
-        self.top_entity = None
-        self.source = source
-        self.parent = parent
-        self.set_origin(parent, url, source, fetchto)
        super(Module, self).__init__()
+        self.init_config(parent, url, source, fetchto)
    def __str__(self):
-        return self.raw_url
+        return self.url
    @property
@@ -68,15 +64,6 @@ class Module(ModuleSynthesis, ModuleOrigin,
        return os.path.dirname(self.path)
-    @property
-    def basename(self):
-        """Get the basename for a module instance"""
-        if self.source == fetch.SVN:
-            return path_mod.svn_basename(self.url)
-        else:
-            return path_mod.url_basename(self.url)
    def submodules(self):
        """Get a list with all the submodules this module instance requires"""
        def __nonull(submodule_list):
@@ -86,7 +73,7 @@ class Module(ModuleSynthesis, ModuleOrigin,
            else:
                return submodule_list
        return __nonull(self.local) + __nonull(self.git) \
-            + __nonull(self.svn) + __nonull(self.git_submodules)
+            + __nonull(self.svn)
    def remove_dir_from_disk(self):
@@ -111,10 +98,6 @@ class Module(ModuleSynthesis, ModuleOrigin,
        """
        logging.debug("Process manifest at: " + os.path.dirname(self.path))
        super(Module, self).process_manifest()
-        #module_list = [ModuleSynthesis, ModuleSimulation,
-        #    ModuleContent, ModuleAltera]
-        #for module_plugin in module_list:
-        #    module_plugin.process_manifest(self)
    def parse_manifest(self):

--- a/hdlmake/module/origin.py
+++ b/hdlmake/module/origin.py
-import os
-import logging
-from .core import ModuleCore
-from hdlmake import fetch
-from hdlmake.util import path as path_mod
-class ModuleOrigin(ModuleCore):
-    def set_origin(self, parent, url, source, fetchto):
-        # Manifest Module Origin Properties
-        self.fetchto = fetchto
-        self.raw_url = url
-        if source != fetch.LOCAL:
-            self.url, self.branch, self.revision = path_mod.url_parse(url)
-            if (
-                    os.path.exists(
-                        os.path.abspath(
-                            os.path.join(fetchto, self.basename)
-                        )
-                    ) and
-                    os.listdir(
-                        os.path.abspath(os.path.join(fetchto, self.basename))
-                    )
-               ):
-                self.path = os.path.abspath(
-                    os.path.join(fetchto, self.basename))
-                self.isfetched = True
-                logging.debug("Module %s (parent: %s) is fetched.",
-                    url, parent.path)
-            else:
-                self.path = None
-                self.isfetched = False
-                logging.debug("Module %s (parent: %s) is NOT fetched.",
-                    url, parent.path)
-        else:
-            self.url, self.branch, self.revision = url, None, None
-            if not os.path.exists(url):
-                logging.error(
-                    "Path to the local module doesn't exist:\n" + url
-                    + "\nThis module was instantiated in: " + str(parent))
-                quit()
-            self.path = url
-            self.isfetched = True
-        #super(ModuleOrigin, self).__init__(parent, url, source, fetchto)
--- a/hdlmake/module_pool.py
+++ b/hdlmake/module_pool.py
@@ -86,12 +86,12 @@ class ModulePool(list):
        """
        from .module import Module
        self._deps_solved = False
-        if url in [m.raw_url for m in self]:
-            return [m for m in self if m.raw_url == url][0]
+        new_module = Module(parent=parent,
-        else:
+                            url=url, source=source,
-            new_module = Module(parent=parent,
+                            fetchto=fetchto)
-                                url=url, source=source,
-                                fetchto=fetchto)
+        if not self.__contains(new_module):
            new_module.set_pool(self)
            self._add(new_module)
            if not self.top_module:
@@ -101,7 +101,8 @@ class ModulePool(list):
                if url:
                    self.top_module.url = url
-            return new_module
+        return new_module
    def _guess_origin(self, path):
        """Guess origin (git, svn, local) of a module at given path"""
@@ -163,7 +164,6 @@ class ModulePool(list):
            new_modules.extend(module.local)
            new_modules.extend(module.svn)
            new_modules.extend(module.git)
-            new_modules.extend(module.git_submodules)
            return new_modules
        fetch_queue = [m for m in self]