vlog_parser.py 25.1 KB
Newer Older
1
#!/usr/bin/python
Paweł Szostek's avatar
Paweł Szostek committed
2
#
Pawel Szostek's avatar
Pawel Szostek committed
3
# Copyright (c) 2013 CERN
4
# Author: Tomasz Wlostowski
5
#         Adrian Fiergolski
Paweł Szostek's avatar
Paweł Szostek committed
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
#
# This file is part of Hdlmake.
#
# Hdlmake is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Hdlmake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Hdlmake.  If not, see <http://www.gnu.org/licenses/>.
#

23 24 25 26
# A Verilog preprocessor. Still lots of stuff to be done,
# but it's already quite useful for calculating dependencies.

"""This module provides the Verilog parser for HDLMake"""
27

28
from __future__ import print_function
29
from __future__ import absolute_import
30 31 32
import os
import re
import sys
33
import logging
34 35 36

from .new_dep_solver import DepParser
from .dep_file import DepRelation
37
from hdlmake.srcfile import create_source_file
38
import six
39

40

41
class VerilogPreprocessor(object):
42

43 44
    """This class provides the Verilog Preprocessor"""

Adrian Fiergolski's avatar
Adrian Fiergolski committed
45
    # Reserved verilog preprocessor keywords. The list is certainly not full
46
    vpp_keywords = [
47
        "default_nettype",
48 49 50 51 52 53 54 55 56 57
        "define",
        "line",
        "include",
        "elsif",
        "ifdef",
        "endif",
        "else",
        "undef",
        "timescale"]

58 59 60
    class VLDefine(object):

        """Class that provides a container for Verilog Defines"""
61

62 63 64 65
        def __init__(self, name, args, expansion):
            self.name = name
            self.args = args
            self.expansion = expansion
66

67 68 69 70
    class VLStack(object):

        """Class that provides a simple binary (true/false) stack
        for Verilog Defines for nested `ifdefs evaluation"""
71

72 73 74
        def __init__(self):
            self.stack = []

75 76 77
        def push(self, v_element):
            """Push element to the stack"""
            self.stack.append(v_element)
78 79

        def pop(self):
80
            "Pop element from the stack"""
81 82 83
            return self.stack.pop()

        def all_true(self):
84 85 86
            """Returns true if the stack is empty or all the contained
            elements are True"""
            return len(self.stack) == 0 or all(self.stack)
87 88

        def flip(self):
89
            """Toggle the following element"""
90 91 92
            self.push(not self.pop())

    def __init__(self):
93
        self.vpp_stack = self.VLStack()
94
        self.vlog_file = None
Adrian Fiergolski's avatar
Adrian Fiergolski committed
95 96 97 98 99 100 101
        # List of `include search paths
        self.vpp_searchdir = ["."]
        # List of macro definitions
        self.vpp_macros = []
        # Dictionary of files sub-included by each file parsed
        self.vpp_filedeps = {}

102
    def _find_macro(self, name):
103 104 105 106
        """Get the Verilog preprocessor macro named 'name'"""
        for macro_aux in self.vpp_macros:
            if macro_aux.name == name:
                return macro_aux
107 108
        return None

109
    def _search_include(self, filename, parent_dir=None):
110 111 112 113
        """Look for the 'filename' Verilog include file in the
        provided 'parent_dir'. If the directory is not provided, the method
        will search for the Verilog include in every defined Verilog
        preprocessor search directory"""
114 115
        if parent_dir is not None:
            possible_file = os.path.join(parent_dir, filename)
116
            if os.path.isfile(possible_file):
117
                return os.path.abspath(possible_file)
118
        for searchdir in self.vlog_file.include_dirs:
119
            probable_file = os.path.join(searchdir, filename)
120
            if os.path.isfile(probable_file):
121
                return os.path.abspath(probable_file)
122 123
        logging.error("Can't find %s for %s in any of the include "
                      "directories: %s", filename, self.vlog_file.file_path,
124
                      ', '.join(self.vlog_file.include_dirs))
125
        sys.exit("\nExiting")
126

127 128 129 130 131 132 133 134
    def _parse_macro_def(self, macro):
        """Parse the provided 'macro' and, if it's not a reserved keyword,
        create a new VLDefine instance and add it to the Verilog preprocessor
        list of macros"""
        name = macro.group(1)
        expansion = macro.group(3)
        if macro.group(2):
            params = macro.group(2).split(",")
135 136 137
        else:
            params = []
        if name in self.vpp_keywords:
138
            logging.error("Attempt to `define a reserved preprocessor keyword")
139
            quit(1)
140
        mdef = self.VLDefine(name, params, expansion)
141 142 143
        self.vpp_macros.append(mdef)
        return mdef

Adrian Fiergolski's avatar
Adrian Fiergolski committed
144
    def _preprocess_file(self, file_content, file_name, library):
145 146 147 148 149 150 151 152 153 154 155 156 157 158
        """Preprocess the content of the Verilog file"""
        def _remove_comment(text):
            """Function that removes the comments from the Verilog code"""
            def replacer(match):
                """Funtion that replace the matching comments"""
                text = match.group(0)
                if text.startswith('/'):
                    return ""
                else:
                    return text
            pattern = re.compile(
                r'//.*?$|/\*.*?\*/|"(?:\\.|[^\\"])*"',
                re.DOTALL | re.MULTILINE)
            return re.sub(pattern, replacer, text)
159

160
        def _degapize(text):
161 162
            """ Create a list in which the verilog sentences are
            stored in an ordered way -- and without empty 'gaps'"""
163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
            lempty = re.compile(r"^\s*$")
            cline = None
            lines = []
            for line_aux in text.splitlines(False):
                if re.match(lempty, line_aux) is not None:
                    continue
                if line_aux.endswith('\\'):
                    if cline is None:
                        cline = ""
                    cline += line_aux[:len(line_aux) - 1]
                    continue
                elif cline:
                    line_aux = cline + line_aux
                    cline = None
                else:
                    cline = None
                lines.append(line_aux)
            return lines
        exps = {"include": re.compile(r"^\s*`include\s+\"(.+)\""),
182
                "define":
183
                re.compile(r"^\s*`define\s+(\w+)(?:\(([\w\s,]*)\))?(.*)"),
184
                "ifdef_elsif":
185
                re.compile(r"^\s*`(ifdef|ifndef|elsif)\s+(\w+)[\s\S]*$"),
186
                "endif_else": re.compile(r"^\s*`(endif|else)\s*$"),
187
                "begin_protected":
188
                re.compile(r"^\s*`pragma\s*protect\s*begin_protected\s*$"),
189
                "end_protected":
190
                re.compile(r"^\s*`pragma\s*protect\s*end_protected\s*$")}
191
        vl_macro_expand = re.compile(r"`(\w+)(?:\(([\w\s,]*)\))?")
192
        # init dependencies
Adrian Fiergolski's avatar
Adrian Fiergolski committed
193
        self.vpp_filedeps[file_name + library] = []
194
        cur_iter = 0
195 196 197
        logging.debug("preprocess file %s (of length %d) in library %s",
                      file_name, len(file_content), library)
        buf = _remove_comment(file_content)
198
        protected_region = False
199 200
        while True:
            new_buf = ""
201 202 203
            n_expansions = 0
            cur_iter += 1
            if cur_iter > 30:
204
                raise Exception("Recursion level exceeded. Nested `includes?")
205
            for line in _degapize(buf):
206 207
                matches = {}
                last = None
208
                for statement, stmt_regex in six.iteritems(exps):
209
                    matches[statement] = re.match(stmt_regex, line)
210
                    if matches[statement]:
211
                        last = matches[statement]
212 213 214 215 216 217 218 219
                if matches["begin_protected"]:
                    protected_region = True
                    continue
                if matches["end_protected"]:
                    protected_region = False
                    continue
                if protected_region:
                    continue
220 221
                if matches["ifdef_elsif"]:
                    cond_true = self._find_macro(last.group(2)) is not None
222
                    if last.group(1) == "ifndef":
223
                        cond_true = not cond_true
224
                    elif last.group(1) == "elsif":
225 226 227
                        self.vpp_stack.pop()
                    self.vpp_stack.push(cond_true)
                    continue
228
                elif matches["endif_else"]:
229
                    if last.group(1) == "endif":
230
                        self.vpp_stack.pop()
231
                    else:  # `else
232 233 234 235 236
                        self.vpp_stack.flip()
                    continue
                if not self.vpp_stack.all_true():
                    continue
                if matches["include"]:
237 238
                    included_file_path = self._search_include(
                        last.group(1), os.path.dirname(file_name))
239 240 241
                    logging.debug("File being parsed %s (library %s) "
                                  "includes %s",
                                  file_name, library, included_file_path)
242 243 244 245 246 247 248 249 250 251 252
                    line = self._preprocess_file(
                        file_content=open(included_file_path, "r").read(),
                        file_name=included_file_path, library=library)
                    self.vpp_filedeps[
                        file_name +
                        library].append(
                        included_file_path)
                    # add the whole include chain to the dependencies of the
                    # currently parsed file
                    self.vpp_filedeps[file_name + library].extend(
                        self.vpp_filedeps[included_file_path + library])
253
                    new_buf += line + '\n'
254
                    n_expansions += 1
255
                    continue
256 257
                elif matches["define"]:
                    self._parse_macro_def(matches["define"])
258

259
                def do_expand(what):
260 261 262
                    """Function to be applied by re.sub to every match of the
                    vl_macro_expand in the Verilof code -- group() returns
                    positive matches as indexed plain strings."""
263
                    if what.group(1) in self.vpp_keywords:
264
                        return '`' + what.group(1)
265 266 267
                    macro = self._find_macro(what.group(1))
                    if macro:
                        return macro.expansion
268
                    else:
269 270 271
                        logging.error("No expansion for macro '`%s' (%s) (%s)",
                                      what.group(1), line[:50]
                                      if len(line) > 50 else line, file_name)
272 273 274 275
                repl_line = re.sub(vl_macro_expand, do_expand, line)
                new_buf += repl_line + '\n'
                # if there was any expansion, then keep on iterating
                if repl_line != line:
276 277
                    n_expansions += 1
            buf = new_buf
278
            if n_expansions == 0:
279 280
                return new_buf

281
    def _define(self, name, expansion):
282 283 284
        """Define a new expansion Verilog macro and add it to the macro
        collection"""
        mdef = self.VLDefine(name, [], expansion)
285 286 287
        self.vpp_macros.append(mdef)

    def add_path(self, path):
288 289
        """Add a new path to the search directory list so that HDLMake
        will search for found includes on it"""
290
        self.vpp_searchdir.append(path)
291

292
    def preprocess(self, vlog_file):
293 294
        """Assign the provided 'vlog_file' to the associated class property
        and then preprocess and return the Verilog code"""
295 296 297 298 299
        # assert isinstance(vlog_file, VerilogFile)
        # assert isinstance(vlog_file, DepFile)
        self.vlog_file = vlog_file
        file_path = vlog_file.file_path
        buf = open(file_path, "r").read()
300 301 302
        return self._preprocess_file(file_content=buf,
                                     file_name=file_path,
                                     library=vlog_file.library)
303 304

    def get_file_deps(self):
305 306
        """Look for all of the defined preprocessor filedeps and return a list
        containing all of them"""
307
        deps = []
308
        for filedep_key in six.iterkeys(self.vpp_filedeps):
309 310
            for filedep in self.vpp_filedeps[filedep_key]:
                deps.append(filedep)
311
        return list(set(deps))
312

313

314
class VerilogParser(DepParser):
315

316 317
    """Class providing the Verilog Parser functionality"""

318
    reserved_words = ["accept_on",
319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
                      "alias",
                      "always",
                      "always_comb",
                      "always_ff",
                      "always_latch",
                      "assert",
                      "assign",
                      "assume",
                      "automatic",
                      "before",
                      "begin",
                      "bind",
                      "bins",
                      "binsof",
                      "bit",
                      "break",
                      "buf",
                      "bufif0",
                      "bufif1",
                      "byte",
                      "case",
                      "casex",
                      "casez",
                      "cell",
                      "chandle",
                      "checker",
                      "class",
                      "clocking",
                      "cmos",
                      "config",
                      "const",
                      "constraint",
                      "context",
                      "continue",
                      "cover",
                      "covergroup",
                      "coverpoint",
                      "cross",
                      "deassign",
                      "default",
                      "defparam",
                      "disable",
                      "dist",
                      "do",
                      "edge",
                      "else",
                      "end",
                      "endcase",
                      "endchecker",
                      "endclass",
                      "endclocking",
                      "endconfig",
                      "endfunction",
                      "endgenerate",
                      "endgroup",
                      "endinterface",
                      "endmodule",
                      "endpackage",
                      "endprimitive",
                      "endprogram",
                      "endproperty",
                      "endsequence",
                      "endspecify",
                      "endtable",
                      "endtask",
                      "enum",
                      "event",
                      "eventually",
                      "expect",
                      "export",
                      "extends",
                      "extern",
                      "final",
                      "first_match",
                      "for",
                      "force",
                      "foreach",
                      "forever",
                      "fork",
                      "forkjoin",
                      "function",
                      "generate",
                      "genvar",
                      "global",
                      "highz0",
                      "highz1",
                      "if",
                      "iff",
                      "ifnone",
                      "ignore_bins",
                      "illegal_bins",
                      "implies",
                      "import",
                      "incdir",
                      "include",
                      "initial",
                      "inout",
                      "input",
                      "inside",
                      "instance",
                      "int",
                      "integer",
                      "interface",
                      "intersect",
                      "join",
                      "join_any",
                      "join_none",
                      "large",
                      "let",
                      "liblist",
                      "library",
                      "local",
                      "localparam",
                      "logic",
                      "longint",
                      "macromodule",
                      "matches",
                      "medium",
                      "modport",
                      "module",
                      "nand",
                      "negedge",
                      "new",
                      "nexttime",
                      "nmos",
                      "nor",
                      "noshowcancelled",
                      "not",
                      "notif0",
                      "notif1",
                      "null",
                      "or",
                      "output",
                      "package",
                      "packed",
                      "parameter",
                      "pmos",
                      "posedge",
                      "primitive",
                      "priority",
                      "program",
                      "property",
                      "protected",
                      "pull0",
                      "pull1",
                      "pulldown",
                      "pullup",
                      "pulsestyle_ondetect",
                      "pulsestyle_onevent",
                      "pure",
                      "rand",
                      "randc",
                      "randcase",
                      "randsequence",
                      "rcmos",
                      "real",
                      "realtime",
                      "ref",
                      "reg",
                      "reject_on",
                      "release",
                      "repeat",
                      "restrict",
                      "return",
                      "rnmos",
                      "rpmos",
                      "rtran",
                      "rtranif0",
                      "rtranif1",
                      "s_always",
                      "scalared",
                      "sequence",
                      "s_eventually",
                      "shortint",
                      "shortreal",
                      "showcancelled",
                      "signed",
                      "small",
                      "s_nexttime",
                      "solve",
                      "specify",
                      "specparam",
                      "static",
                      "string",
                      "strong",
                      "strong0",
                      "strong1",
                      "struct",
                      "s_until",
                      "super",
                      "supply0",
                      "supply1",
                      "sync_accept_on",
                      "sync_reject_on",
                      "table",
                      "tagged",
                      "task",
                      "this",
                      "throughout",
                      "time",
                      "timeprecision",
                      "timeunit",
                      "tran",
                      "tranif0",
                      "tranif1",
                      "tri",
                      "tri0",
                      "tri1",
                      "triand",
                      "trior",
                      "trireg",
                      "type",
                      "typedef",
                      "union",
                      "unique",
                      "unique0",
                      "unsigned",
                      "until",
                      "until_with",
                      "untypted",
                      "use",
                      "var",
                      "vectored",
                      "virtual",
                      "void",
                      "wait",
                      "wait_order",
                      "wand",
                      "weak",
                      "weak0",
                      "weak1",
                      "while",
                      "wildcard",
                      "wire",
                      "with",
                      "within",
                      "wor",
                      "xnor",
                      "xor"]

    def __init__(self, dep_file):
        DepParser.__init__(self, dep_file)
        self.preprocessor = VerilogPreprocessor()
562
        self.preprocessed = False
563 564

    def add_search_path(self, path):
565 566
        """Add a new candidate path to the Verilog preprocessor list
        containing the include dir candidates"""
567
        self.preprocessor.add_path(path)
568

569
    def parse(self, dep_file):
570 571
        """Parse the provided Verilog file and add to its properties
        all of the detected dependency relations"""
572 573
        if dep_file.is_parsed:
            return
574
        logging.debug("Parsing %s", dep_file.path)
575 576
        # assert isinstance(dep_file, DepFile), print("unexpected type: " +
        # str(type(dep_file)))
577 578
        buf = self.preprocessor.preprocess(dep_file)
        self.preprocessed = buf[:]
579
        # add includes as dependencies
580
        try:
581 582
            includes = self.preprocessor.vpp_filedeps[
                dep_file.path + dep_file.library]
583
            for file_aux in includes:
584
                dep_file.depends_on.add(
585
                    create_source_file(path=file_aux,
586 587
                                       module=dep_file.module,
                                       is_include=True))
588 589
            logging.debug("%s has %d includes.",
                          str(dep_file), len(includes))
590 591
        except KeyError:
            logging.debug(str(dep_file) + " has no includes.")
592 593
        # look for packages used inside in file
        # it may generate false dependencies as package in SV can be used by:
594
        #    import my_package::*;
595
        # or directly
596
        #    logic var = my_package::MY_CONST;
597 598
        # The same way constants and others can be imported directly from
        # other modules:
599
        #    logic var = my_other_module::MY_CONST;
600 601
        # and HdlMake will anyway create dependency marking my_other_module as
        # requested package
602
        import_pattern = re.compile(r"(\w+) *::(\w+|\\*)")
603

604 605 606 607 608 609 610
        def do_imports(text):
            """Function to be applied by re.subn to every match of the
            import_pattern in the Verilog code -- group() returns positive
            matches as indexed plain strings. It adds the found USE
            relations to the file"""
            logging.debug("file %s imports/uses %s.%s package",
                          dep_file.path, dep_file.library, text.group(1))
611
            dep_file.add_relation(
612 613
                DepRelation("%s.%s" % (dep_file.library, text.group(1)),
                            DepRelation.USE, DepRelation.PACKAGE))
614
        re.subn(import_pattern, do_imports, buf)
615 616
        # packages
        m_inside_package = re.compile(
617
            r"package\s+(\w+)\s*(?:\(.*?\))?\s*(.+?)endpackage",
618
            re.DOTALL | re.MULTILINE)
619

620 621 622 623 624 625
        def do_package(text):
            """Function to be applied by re.subn to every match of the
            m_inside_pattern in the Verilog code -- group() returns positive
            matches as indexed plain strings. It adds the found PROVIDE
            relations to the file"""
            logging.debug("found pacakge %s.%s", dep_file.library,
626
                          text.group(1))
627
            dep_file.add_relation(
628 629
                DepRelation("%s.%s" % (dep_file.library, text.group(1)),
                            DepRelation.PROVIDE, DepRelation.PACKAGE))
630
        re.subn(m_inside_package, do_package, buf)
631 632
        # modules and instatniations
        m_inside_module = re.compile(
633 634
            r"(?:module|interface)\s+(\w+)\s*(?:\(.*?\))?\s*(.+?)"
            r"(?:endmodule|endinterface)",
635 636
            re.DOTALL | re.MULTILINE)
        m_instantiation = re.compile(
637
            r"(?:\A|\s*)\s*(\w+)\s+(?:#\s*\(.*?\)\s*)?(\w+)\s*\(.*?\)\s*",
638
            re.DOTALL | re.MULTILINE)
639

640 641 642 643 644 645
        def do_module(text):
            """Function to be applied by re.sub to every match of the
            m_inside_module in the Verilog code -- group() returns
            positive  matches as indexed plain strings. It adds the found
            PROVIDE relations to the file"""
            logging.debug("found module %s.%s", dep_file.library,
646
                          text.group(1))
647
            dep_file.add_relation(
648 649
                DepRelation("%s.%s" % (dep_file.library, text.group(1)),
                            DepRelation.PROVIDE, DepRelation.MODULE))
650

651 652 653 654 655 656 657
            def do_inst(text):
                """Function to be applied by re.sub to every match of the
                m_instantiation in the Verilog code -- group() returns positive
                matches as indexed plain strings. It adds the found USE
                relations to the file"""
                mod_name = text.group(1)
                if mod_name in self.reserved_words:
658
                    return
659 660
                logging.debug("-> instantiates %s.%s as %s",
                              dep_file.library, text.group(1), text.group(2))
661
                dep_file.add_relation(
662 663 664
                    DepRelation("%s.%s" % (dep_file.library, text.group(1)),
                                DepRelation.USE, DepRelation.MODULE))
            re.subn(m_instantiation, do_inst, text.group(2))
665 666 667 668 669 670
        re.subn(m_inside_module, do_module, buf)
        dep_file.add_relation(
            DepRelation(
                dep_file.path,
                DepRelation.PROVIDE,
                DepRelation.INCLUDE))
671
        dep_file.is_parsed = True