Issue backfiler (#2622)

* Fixing f-string reported by lint

* Add a backfiler

Author: DonggeLiu

butler.py

@@ -44,7 +44,7 @@ def __init__(self, *args, **kwargs):
 
   def error(self, message):
     """Override to print full help for ever error."""
-    sys.stderr.write('error: %s\n' % message)
+    sys.stderr.write(f'error: {message}\n')
     self.print_help()
     sys.exit(2)
 
@@ -190,6 +190,8 @@ def main():
   parser_run.add_argument(
       'script_name',
       help='The script module name under `./local/butler/scripts`.')
+  parser_run.add_argument(
+      '--script_args', action='append', help='Script specific arguments')
   parser_run.add_argument(
       '--non-dry-run',
       action='store_true',
@@ -299,7 +301,7 @@ def main():
     return
 
   _setup()
-  command = importlib.import_module('local.butler.%s' % args.command)
+  command = importlib.import_module(f'local.butler.{args.command}')
   command.execute(args)
 
 

src/local/butler/run.py

@@ -16,6 +16,7 @@
 
 import importlib
 import os
+import sys
 
 from local.butler import constants
 from src.clusterfuzz._internal.config import local_config
@@ -25,6 +26,8 @@
 def execute(args):
   """Run Python unit tests under v2. For unittests involved appengine, sys.path
      needs certain modification."""
+  print(args.script_args)
+  sys.path.insert(0, os.path.abspath(os.path.join('src', 'appengine')))
   os.environ['CONFIG_DIR_OVERRIDE'] = args.config_dir
   local_config.ProjectConfig().set_environment()
 
@@ -39,8 +42,7 @@ def execute(args):
           'For permanent modifications, re-run with --non-dry-run.')
 
   with ndb_init.context():
-    script = importlib.import_module(
-        'local.butler.scripts.%s' % args.script_name)
+    script = importlib.import_module(f'local.butler.scripts.{args.script_name}')
     script.execute(args)
 
   if not args.local:

src/local/butler/scripts/backfiler.py

@@ -0,0 +1,42 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""backfiler.py queries past testcases of the given projects
+  and filed them to the projects' GitHub repository."""
+
+from clusterfuzz._internal.datastore import data_types
+from clusterfuzz._internal.datastore import ndb_utils
+from libs.issue_management import oss_fuzz_github
+
+
+def execute(args):
+  """Query Testcases of the given projects,
+    and conditionally file them to the corresponding GitHub repo."""
+  if not args.script_args:
+    print('Need at least one project name (with -p) '
+          'when running the backfiler.')
+    return
+  for project_name in args.script_args:
+    print(f'Back filing project {project_name}')
+    for testcase in data_types.Testcase.query(
+        ndb_utils.is_true(data_types.Testcase.open),
+        ndb_utils.is_false(data_types.Testcase.one_time_crasher_flag),
+        data_types.Testcase.status == 'Processed',
+        data_types.Testcase.project_name == project_name,
+    ):
+      if not testcase.bug_information:
+        print(f'Skip testcase without bugs: {testcase.key.id()}')
+        continue
+      print(f'Back filing testcase id: {testcase.key.id()}')
+      if args.non_dry_run:
+        oss_fuzz_github.file_issue(testcase)