fbt: compile_db fixes (#1981)

* fbt: forked compilation_db tool
* fbt: fixes for static analysis
* pvs-studio: ignoring more generic warnings
* fbt: util: added extract_abs_dir
* vscode: added fap-set-debug-elf-root for debug configurations

.github/workflows/pvs_studio.yml

@@ -57,7 +57,7 @@ jobs:
 
       - name: 'Generate compile_comands.json'
         run: |
-          FBT_TOOLCHAIN_PATH=/runner/_work ./fbt COMPACT=1 version_json proto_ver icons firmware_cdb dolphin_internal dolphin_blocking
+          FBT_TOOLCHAIN_PATH=/runner/_work ./fbt COMPACT=1 version_json proto_ver icons firmware_cdb dolphin_internal dolphin_blocking _fap_icons
 
       - name: 'Static code analysis'
         run: |

.gitignore

@@ -54,3 +54,5 @@ openocd.log
 # PVS Studio temporary files
 .PVS-Studio/
 PVS-Studio.log
+
+.gdbinit

.pvsconfig

@@ -5,6 +5,7 @@
 //-V:BPTREE_DEF2:779,1086,557,773,512
 //-V:DICT_DEF2:779,524,776,760,1044,1001,729,590,568,747,685
 //-V:ALGO_DEF:1048,747,1044
+//-V:TUPLE_DEF2:524,590,1001,760
 
 # Non-severe malloc/null pointer deref warnings
 //-V::522:2,3

.vscode/example/launch.json

@@ -38,6 +38,7 @@
             "postAttachCommands": [
                 // "compare-sections",
                 "source debug/flipperapps.py",
+                "fap-set-debug-elf-root build/latest/.extapps",
                 // "source debug/FreeRTOS/FreeRTOS.py",
                 // "svd_load debug/STM32WB55_CM4.svd"
             ]
@@ -59,6 +60,7 @@
                 "set confirm off",
                 "set mem inaccessible-by-default off",
                 "source debug/flipperapps.py",
+                "fap-set-debug-elf-root build/latest/.extapps",
                 // "compare-sections",
             ]
             // "showDevDebugOutput": "raw",
@@ -76,6 +78,7 @@
             "rtos": "FreeRTOS",
             "postAttachCommands": [
                 "source debug/flipperapps.py",
+                "fap-set-debug-elf-root build/latest/.extapps",
             ]
             // "showDevDebugOutput": "raw",
         },
@@ -95,6 +98,7 @@
             ],
             "postAttachCommands": [
                 "source debug/flipperapps.py",
+                "fap-set-debug-elf-root build/latest/.extapps",
             ],
             // "showDevDebugOutput": "raw",
         },

SConstruct

@@ -200,7 +200,9 @@ firmware_debug = distenv.PhonyTarget(
     source=firmware_env["FW_ELF"],
     GDBOPTS="${GDBOPTS_BASE}",
     GDBREMOTE="${OPENOCD_GDB_PIPE}",
-    FBT_FAP_DEBUG_ELF_ROOT=firmware_env.subst("$FBT_FAP_DEBUG_ELF_ROOT"),
+    FBT_FAP_DEBUG_ELF_ROOT=firmware_env.subst("$FBT_FAP_DEBUG_ELF_ROOT").replace(
+        "\\", "/"
+    ),
 )
 distenv.Depends(firmware_debug, firmware_flash)
 
@@ -210,7 +212,9 @@ distenv.PhonyTarget(
     source=firmware_env["FW_ELF"],
     GDBOPTS="${GDBOPTS_BASE} ${GDBOPTS_BLACKMAGIC}",
     GDBREMOTE="${BLACKMAGIC_ADDR}",
-    FBT_FAP_DEBUG_ELF_ROOT=firmware_env.subst("$FBT_FAP_DEBUG_ELF_ROOT"),
+    FBT_FAP_DEBUG_ELF_ROOT=firmware_env.subst("$FBT_FAP_DEBUG_ELF_ROOT").replace(
+        "\\", "/"
+    ),
 )
 
 # Debug alien elf

firmware.scons

@@ -264,7 +264,10 @@ fw_artifacts.extend(
 
 fwcdb = fwenv.CompilationDatabase()
 # without filtering, both updater & firmware commands would be generated in same file
-fwenv.Replace(COMPILATIONDB_PATH_FILTER=fwenv.subst("*${FW_FLAVOR}*"))
+fwenv.Replace(
+    COMPILATIONDB_PATH_FILTER=fwenv.subst("*${FW_FLAVOR}*"),
+    COMPILATIONDB_SRCPATH_FILTER="*.c*",
+)
 AlwaysBuild(fwcdb)
 Precious(fwcdb)
 NoClean(fwcdb)

scripts/fbt/util.py

@@ -43,12 +43,18 @@ def single_quote(arg_list):
     return " ".join(f"'{arg}'" if " " in arg else str(arg) for arg in arg_list)
 
 
-def extract_abs_dir_path(node):
+def extract_abs_dir(node):
     if isinstance(node, SCons.Node.FS.EntryProxy):
         node = node.get()
 
     for repo_dir in node.get_all_rdirs():
         if os.path.exists(repo_dir.abspath):
-            return repo_dir.abspath
+            return repo_dir
+
+
+def extract_abs_dir_path(node):
+    abs_dir_node = extract_abs_dir(node)
+    if abs_dir_node is None:
+        raise StopError(f"Can't find absolute path for {node.name}")
 
-    raise StopError(f"Can't find absolute path for {node.name}")
+    return abs_dir_node.abspath

scripts/fbt_tools/compilation_db.py

@@ -0,0 +1,278 @@
+"""
+Implements the ability for SCons to emit a compilation database for the MongoDB project. See
+http://clang.llvm.org/docs/JSONCompilationDatabase.html for details on what a compilation
+database is, and why you might want one. The only user visible entry point here is
+'env.CompilationDatabase'. This method takes an optional 'target' to name the file that
+should hold the compilation database, otherwise, the file defaults to compile_commands.json,
+which is the name that most clang tools search for by default.
+"""
+
+# Copyright 2020 MongoDB Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+import json
+import itertools
+import fnmatch
+import SCons
+
+from SCons.Tool.cxx import CXXSuffixes
+from SCons.Tool.cc import CSuffixes
+from SCons.Tool.asm import ASSuffixes, ASPPSuffixes
+
+# TODO: Is there a better way to do this than this global? Right now this exists so that the
+# emitter we add can record all of the things it emits, so that the scanner for the top level
+# compilation database can access the complete list, and also so that the writer has easy
+# access to write all of the files. But it seems clunky. How can the emitter and the scanner
+# communicate more gracefully?
+__COMPILATION_DB_ENTRIES = []
+
+
+# We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even
+# integrate with the cache, but there doesn't seem to be much call for it.
+class __CompilationDbNode(SCons.Node.Python.Value):
+    def __init__(self, value):
+        SCons.Node.Python.Value.__init__(self, value)
+        self.Decider(changed_since_last_build_node)
+
+
+def changed_since_last_build_node(child, target, prev_ni, node):
+    """Dummy decider to force always building"""
+    return True
+
+
+def make_emit_compilation_DB_entry(comstr):
+    """
+    Effectively this creates a lambda function to capture:
+    * command line
+    * source
+    * target
+    :param comstr: unevaluated command line
+    :return: an emitter which has captured the above
+    """
+    user_action = SCons.Action.Action(comstr)
+
+    def emit_compilation_db_entry(target, source, env):
+        """
+        This emitter will be added to each c/c++ object build to capture the info needed
+        for clang tools
+        :param target: target node(s)
+        :param source: source node(s)
+        :param env: Environment for use building this node
+        :return: target(s), source(s)
+        """
+
+        dbtarget = __CompilationDbNode(source)
+
+        entry = env.__COMPILATIONDB_Entry(
+            target=dbtarget,
+            source=[],
+            __COMPILATIONDB_UOUTPUT=target,
+            __COMPILATIONDB_USOURCE=source,
+            __COMPILATIONDB_UACTION=user_action,
+            __COMPILATIONDB_ENV=env,
+        )
+
+        # TODO: Technically, these next two lines should not be required: it should be fine to
+        # cache the entries. However, they don't seem to update properly. Since they are quick
+        # to re-generate disable caching and sidestep this problem.
+        env.AlwaysBuild(entry)
+        env.NoCache(entry)
+
+        __COMPILATION_DB_ENTRIES.append(dbtarget)
+
+        return target, source
+
+    return emit_compilation_db_entry
+
+
+def compilation_db_entry_action(target, source, env, **kw):
+    """
+    Create a dictionary with evaluated command line, target, source
+    and store that info as an attribute on the target
+    (Which has been stored in __COMPILATION_DB_ENTRIES array
+    :param target: target node(s)
+    :param source: source node(s)
+    :param env: Environment for use building this node
+    :param kw:
+    :return: None
+    """
+
+    command = env["__COMPILATIONDB_UACTION"].strfunction(
+        target=env["__COMPILATIONDB_UOUTPUT"],
+        source=env["__COMPILATIONDB_USOURCE"],
+        env=env["__COMPILATIONDB_ENV"],
+    )
+
+    entry = {
+        "directory": env.Dir("#").abspath,
+        "command": command,
+        "file": env["__COMPILATIONDB_USOURCE"][0],
+        "output": env["__COMPILATIONDB_UOUTPUT"][0],
+    }
+
+    target[0].write(entry)
+
+
+def write_compilation_db(target, source, env):
+    entries = []
+
+    use_abspath = env["COMPILATIONDB_USE_ABSPATH"] in [True, 1, "True", "true"]
+    use_path_filter = env.subst("$COMPILATIONDB_PATH_FILTER")
+    use_srcpath_filter = env.subst("$COMPILATIONDB_SRCPATH_FILTER")
+
+    for s in __COMPILATION_DB_ENTRIES:
+        entry = s.read()
+        source_file = entry["file"]
+        output_file = entry["output"]
+
+        if source_file.rfile().srcnode().exists():
+            source_file = source_file.rfile().srcnode()
+
+        if use_abspath:
+            source_file = source_file.abspath
+            output_file = output_file.abspath
+        else:
+            source_file = source_file.path
+            output_file = output_file.path
+
+        # print("output_file, path_filter", output_file, use_path_filter)
+        if use_path_filter and not fnmatch.fnmatch(output_file, use_path_filter):
+            continue
+
+        if use_srcpath_filter and not fnmatch.fnmatch(source_file, use_srcpath_filter):
+            continue
+
+        path_entry = {
+            "directory": entry["directory"],
+            "command": entry["command"],
+            "file": source_file,
+            "output": output_file,
+        }
+
+        entries.append(path_entry)
+
+    with open(target[0].path, "w") as output_file:
+        json.dump(
+            entries, output_file, sort_keys=True, indent=4, separators=(",", ": ")
+        )
+
+
+def scan_compilation_db(node, env, path):
+    return __COMPILATION_DB_ENTRIES
+
+
+def compilation_db_emitter(target, source, env):
+    """fix up the source/targets"""
+
+    # Someone called env.CompilationDatabase('my_targetname.json')
+    if not target and len(source) == 1:
+        target = source
+
+    # Default target name is compilation_db.json
+    if not target:
+        target = [
+            "compile_commands.json",
+        ]
+
+    # No source should have been passed. Drop it.
+    if source:
+        source = []
+
+    return target, source
+
+
+def generate(env, **kwargs):
+    static_obj, shared_obj = SCons.Tool.createObjBuilders(env)
+
+    env.SetDefault(
+        COMPILATIONDB_COMSTR=kwargs.get(
+            "COMPILATIONDB_COMSTR", "Building compilation database $TARGET"
+        ),
+        COMPILATIONDB_USE_ABSPATH=False,
+        COMPILATIONDB_PATH_FILTER="",
+        COMPILATIONDB_SRCPATH_FILTER="",
+    )
+
+    components_by_suffix = itertools.chain(
+        itertools.product(
+            CSuffixes,
+            [
+                (static_obj, SCons.Defaults.StaticObjectEmitter, "$CCCOM"),
+                (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCCCOM"),
+            ],
+        ),
+        itertools.product(
+            CXXSuffixes,
+            [
+                (static_obj, SCons.Defaults.StaticObjectEmitter, "$CXXCOM"),
+                (shared_obj, SCons.Defaults.SharedObjectEmitter, "$SHCXXCOM"),
+            ],
+        ),
+        itertools.product(
+            ASSuffixes,
+            [(static_obj, SCons.Defaults.StaticObjectEmitter, "$ASCOM")],
+            [(shared_obj, SCons.Defaults.SharedObjectEmitter, "$ASCOM")],
+        ),
+        itertools.product(
+            ASPPSuffixes,
+            [(static_obj, SCons.Defaults.StaticObjectEmitter, "$ASPPCOM")],
+            [(shared_obj, SCons.Defaults.SharedObjectEmitter, "$ASPPCOM")],
+        ),
+    )
+
+    for entry in components_by_suffix:
+        suffix = entry[0]
+        builder, base_emitter, command = entry[1]
+
+        # Assumes a dictionary emitter
+        emitter = builder.emitter.get(suffix, False)
+        if emitter:
+            # We may not have tools installed which initialize all or any of
+            # cxx, cc, or assembly. If not skip resetting the respective emitter.
+            builder.emitter[suffix] = SCons.Builder.ListEmitter(
+                [
+                    emitter,
+                    make_emit_compilation_DB_entry(command),
+                ]
+            )
+
+    env.Append(
+        BUILDERS={
+            "__COMPILATIONDB_Entry": SCons.Builder.Builder(
+                action=SCons.Action.Action(compilation_db_entry_action, None),
+            ),
+            "CompilationDatabase": SCons.Builder.Builder(
+                action=SCons.Action.Action(
+                    write_compilation_db, "$COMPILATIONDB_COMSTR"
+                ),
+                target_scanner=SCons.Scanner.Scanner(
+                    function=scan_compilation_db, node_class=None
+                ),
+                emitter=compilation_db_emitter,
+                suffix="json",
+            ),
+        }
+    )
+
+
+def exists(env):
+    return True