[IMP] runbot: add support for check based on semgrep rules

runbot/models/__init__.py

@@ -25,6 +25,7 @@
 from . import res_config_settings
 from . import res_users
 from . import runbot
+from . import semgrep_rule
 from . import team
 from . import upgrade
 from . import user

runbot/models/build.py

@@ -982,7 +982,7 @@ def _docker_run(self, step, cmd=None, ro_volumes=None, env_variables=None, **kwa
         for dest, source in _ro_volumes.items():
             ro_volumes[f'/data/build/{dest}'] = source
         if 'image_tag' not in kwargs:
-            kwargs.update({'image_tag': self.params_id.dockerfile_id.image_tag})
+            kwargs.update({'image_tag':step.dockerfile_id.image_tag or self.params_id.dockerfile_id.image_tag})
         dockerfile_variant = self.params_id.config_data.get('dockerfile_variant', step.dockerfile_variant)
         if dockerfile_variant and f'.{dockerfile_variant.lower()}' not in kwargs['image_tag']:
             kwargs['image_tag'] += f'.{dockerfile_variant.lower()}'

runbot/models/build_config.py

@@ -355,6 +355,7 @@ class ConfigStepUpgradeDb(models.Model):
         ('test_upgrade', 'Test Upgrade'),
         ('restore', 'Restore'),
         ('dynamic', 'Dynamic'),
+        ('semgrep,', 'Semgrep')
     ]
 
 
@@ -377,6 +378,7 @@ class ConfigStep(models.Model):
     group_name = fields.Char('Group name', related='group.name')
     make_stats = fields.Boolean('Make stats', default=False)
     build_stat_regex_ids = fields.Many2many('runbot.build.stat.regex', string='Stats Regexes')
+    dockerfile_id = fields.Many2one('runbot.dockerfile', string='Dockerfile')
     dockerfile_variant = fields.Char('Docker Variant')
     # install_odoo
     create_db = fields.Boolean('Create Db', default=True, tracking=True)  # future
@@ -427,6 +429,10 @@ class ConfigStep(models.Model):
     restore_download_db_suffix = fields.Char('Download db suffix')
     restore_rename_db_suffix = fields.Char('Rename db suffix')
 
+    semgrep_category = fields.Many2one('runbot.checker_category', string='Semgrep Category', tracking=True)
+    custom_link = fields.Char('Custom link for semgrep codes', tracking=True)
+    disable_nosem = fields.Boolean('Disable Semgrep', default=False, tracking=True)
+
     commit_limit = fields.Integer('Commit limit', default=50)
     file_limit = fields.Integer('File limit', default=450)
     break_before_if_ko = fields.Boolean('Break before this step if build is ko')
@@ -1232,6 +1238,8 @@ def _make_results(self, build):
             self._make_upgrade_results(build)
         elif active_job_type == 'restore':
             self._make_restore_results(build)
+        elif active_job_type == 'semgrep':
+            self._make_semgrep_results(build)
 
     def _make_python_results(self, build):
         eval_ctx = self._make_python_ctx(build)
@@ -1645,6 +1653,139 @@ def consume_remaining_tasks(self, build):
             return next_index < len(steps)
         return False
 
+    def _run_semgrep(self, build):
+        if not self._check_limits(build):
+            return
+
+        rules = self.env['runbot.semgrep_rule'].search([
+            ("category_id", '=', self.semgrep_category.id),
+            '|', ("min_version", '=', False), ("min_version.number", "<=", build.params_id.version_id.number),
+            '|', ('max_version', '=', False), ('max_version.number', '>', build.params_id.version_id.number),
+        ])
+        if not rules:
+            return
+
+        for rule in rules:
+            build._write_file(f"rules/{rule.x_name}.yaml", "rules:\n" + rule.x_rule_text)
+
+        exports = build._checkout()
+
+        files = []
+        targets = []
+        for link in build.params_id.commit_link_ids:
+            # filtering section for progressive CI (style & security)
+            modified = link.commit_id.repo_id._git([
+                'diff',
+                '%s..%s' % (link.merge_base_commit_id.name, link.commit_id.name),
+                '--',
+                '*.py',
+                '*.js',
+            ])
+            for patched_file in PatchSet(modified.splitlines(keepends=True)):
+                target = patched_file.target_file.removeprefix('b/')
+                if target.startswith(('setup/',)):
+                    continue
+                target = link.commit_id.repo_id.name + '/' + target
+
+                before = len(targets)
+                targets.extend(
+                    f"{target}:{line.target_line_no}"
+                    for hunk in patched_file
+                    for line in hunk
+                    if line.is_added
+                )
+                # only look at file if it has additions
+                if len(targets) > before:
+                    files.append(target)
+
+        if not files:
+            build._log("", "Nothing to scan.")
+            return
+
+        build._log("", f"checking {len(targets)} lines in {len(files)} files")
+
+        # add empty ignore file, otherwise semgrep ignores test directories by default
+        build._write_file(".semgrepignore", "")
+        build._write_file(f"logs/{self.name}-files_list.txt", "\n".join(files))
+        build._write_file("targets", "\n".join(targets))
+
+        cmd = f"semgrep scan {'--disable-nosem' if self.disable_nosem else ''} -c /data/build/rules --json --timeout=0 --verbose $(cat logs/{self.name}-files_list.txt) > /data/build/results.json"
+
+        return {
+            "cmd": cmd,
+            "container_name": build._get_docker_name(),
+            "ro_volumes": exports,
+        }
+
+    def _make_semgrep_results(self, build):
+        step_result = "ok"
+        if build._is_file("targets"):
+            targets = set(build._read_file("targets").splitlines(keepends=False))
+            f = build._read_file("results.json")
+            semgrep_result = json.loads(f) if f else {}
+        else:
+            targets = set()
+            semgrep_result = {}
+
+        repo = {
+            link.commit_id.repo_id.name: (link.branch_id.remote_id.base_url, link.commit_id)
+            for link in build.params_id.commit_link_ids
+        }
+
+        # some of the lints can catch the same issue multiple times on the same line, and semgrep does not dedup
+        seen = set()
+
+        # rules results
+        for result in semgrep_result.get('results', ()):
+            _, _, code = result['check_id'].rpartition('.')
+            start = result['start']['line']
+            matches = targets & {
+                f"{result['path']}:{start}"
+                for line in range(result['start']['line'], result['end']['line']+1)
+            }
+            if not matches:
+                continue
+
+            if all((target, code) in seen for target in matches):
+                continue
+            seen.update((target, code) for target in matches)
+
+            repo_name, path = result['path'].split('/', 1)
+            filename = f"{path}:{start}"
+            repo_base_url, commit = repo[repo_name]
+            commit_hash = commit.name
+
+            # FIXME: should be a code block :(
+            extra = result['extra']
+            #snippet = extra['lines'] #"\n".join(f'{line}' for line in extra['lines'].splitlines(keepends=False))
+            file = commit._read_source(path, mode='rb')
+            snippet = file[result['start']['offset']:result['end']['offset']].decode()
+
+            codelink = f"{code}: {extra['message']}\n"
+            if self.custom_link:
+                # message may be sensitive, do not display, show snippet on same line if single line, otherwise block below
+                codelink = f"[{code} 🔗]({self.custom_link}#{code}): "
+            if '\n' in snippet:
+                snippet = '\n' + snippet
+
+            build._log(
+                "semgrep",
+                f"""\
+        [%s](https://%s/blob/%s/%s#L%s-L%s)
+        {codelink}`%s`
+            """, filename, repo_base_url, commit_hash , path, result['start']['line'], result['end']['line'], snippet,
+                level=extra['severity'],
+                log_type='markdown',
+            )
+            if extra['severity'] != 'INFO':
+                step_result = "ko"
+
+        # internal semgrep errors
+        for err in semgrep_result.get('errors', ()):
+            build._log("semgrep", err.get('message') or str(err), log_type='markdown')
+
+        build['local_result'] = build._get_worst_result([build.local_result, step_result])
+
 
 class ConfigStepOrder(models.Model):
     _name = 'runbot.build.config.step.order'

runbot/models/semgrep_rule.py

@@ -0,0 +1,64 @@
+from odoo import api, fields, models
+
+class SemgrepRule(models.Model):
+    _name = 'runbot.semgrep_rule'
+    _description = 'Semgrep Rule'
+
+    name = fields.Char(string='Rule Name', required=True)
+    category_id = fields.Many2one('runbot.checker_category', string='Category', required=True, index=True)
+    language = fields.Selection([('python', 'Python'), ('javascript', 'JavaScript'), ('generic', 'Generic')], required=True)
+    max_version = fields.Many2one('runbot.version', string='Max Odoo Version', help='Maximum exclusive Odoo version this rule applies to')
+    min_version = fields.Many2one('runbot.version', string='Min Odoo Version', help='Minimum inclusive Odoo version this rule applies to')
+    message = fields.Char(string='Error message', help='Message to display when the rule is triggered', required=True)
+    rule = fields.Text("Rule", required=True)
+    rule_text = fields.Text("Rule Text", compute='_compute_rule_text')
+    severity = fields.Selection([('INFO', 'INFO'), ('WARNING', 'WARNING'), ('ERROR', 'ERROR')], string='Severity', required=True)
+
+
+    @api.depends('name', 'message', 'severity', 'language', 'rule')
+    def _compute_rule_text(self):
+        def indent_by(s, by=2):
+            indent = " " * by
+            return ''.join(
+                l if l.isspace() else indent + l
+                for l in s.splitlines(keepends=True)
+            )
+
+        def count_indent(s):
+            for line in s.splitlines(keepends=False):
+                if line.isspace():
+                    continue
+                return len(line) - len(line.lstrip())
+            return None
+
+        self.x_rule_text = ''
+        for r in self:
+            rule = r.x_rule
+            if not rule:
+                continue
+
+            indent = count_indent(rule)
+            if indent is None:
+                continue
+
+            if indent < 2:
+                rule = indent_by(rule, 2-indent)
+                indent = 2
+
+            i_indent = " " * (indent - 2)
+            s_indent = " " * indent
+            r.rule_text = f"""\
+{i_indent}- id: {r.x_name}
+{s_indent}languages: [{r.x_language}]
+{s_indent}severity: {r.x_severity}
+{s_indent}message: {r.x_message!r}
+{rule}
+ """
+
+
+
+class CheckerCategory(models.Model):
+    _name = 'runbot.checker_category'
+    _description = 'Checker Category'
+
+    name = fields.Char(string='Category Name', required=True)

runbot/views/config_views.xml

@@ -47,6 +47,7 @@
                         <field name="job_type"/>
                         <field name="make_stats"/>
                         <field name="protected"/>
+                        <field name="dockerfile_id"/>
                         <field name="dockerfile_variant"/>
                         <field name="default_sequence" groups="base.group_no_one"/>
                         <field name="group" groups="base.group_no_one"/>
@@ -120,6 +121,12 @@
                     <group string="Codeowner settings" invisible="job_type not in ('codeowner', 'python', 'dynamic')">
                         <field name="fallback_reviewer"/>
                     </group>
+
+                    <group string="Semgrep settings" invisible="job_type not in ('semgrep', 'python', 'dynamic')">
+                        <field name="semgrep_category"/>
+                        <field name="disable_nosem"/>
+                        <field name="custom_link"/>
+                    </group>
                     <group>
                       <field name="step_order_ids" groups="base.group_no_one" readonly="1">
                         <list>