Update yapf docs and run auto-formater for modified files.

Author: mingxingtan

efficientdet/dataset/create_coco_tfrecord.py

@@ -45,7 +45,6 @@
 from dataset import label_map_util
 from dataset import tfrecord_util
 
-
 flags.DEFINE_boolean(
     'include_masks', False, 'Whether to include instance segmentations masks '
     '(PNG encoded) in the result. default: False.')
@@ -288,9 +287,8 @@ def _create_tf_record_from_coco_annotations(images_info_file,
 
   logging.info('writing to output path: %s', output_path)
   writers = [
-      tf.python_io.TFRecordWriter(
-          output_path + '-%05d-of-%05d.tfrecord' % (i, num_shards))
-      for i in range(num_shards)
+      tf.python_io.TFRecordWriter(output_path + '-%05d-of-%05d.tfrecord' %
+                                  (i, num_shards)) for i in range(num_shards)
   ]
   images = _load_images_info(images_info_file)
 
@@ -319,10 +317,11 @@ def _get_caption_annotation(image_id):
   pool = multiprocessing.Pool(FLAGS.num_threads)
   total_num_annotations_skipped = 0
   for idx, (_, tf_example, num_annotations_skipped) in enumerate(
-      pool.imap(_pool_create_tf_example,
-                [(image, image_dir, _get_object_annotation(image['id']),
-                  category_index, _get_caption_annotation(image['id']),
-                  include_masks) for image in images])):
+      pool.imap(
+          _pool_create_tf_example,
+          [(image, image_dir, _get_object_annotation(image['id']),
+            category_index, _get_caption_annotation(image['id']), include_masks)
+           for image in images])):
     if idx % 100 == 0:
       logging.info('On image %d of %d', idx, len(images))
 

efficientdet/dataset/create_pascal_tfrecord.py

@@ -37,7 +37,6 @@
 
 from dataset import tfrecord_util
 
-
 flags.DEFINE_string('data_dir', '', 'Root directory to raw PASCAL VOC dataset.')
 flags.DEFINE_string('set', 'train', 'Convert training set, validation set or '
                     'merged set.')
@@ -57,13 +56,29 @@
 YEARS = ['VOC2007', 'VOC2012', 'merged']
 
 pascal_label_map_dict = {
-    'background': 0, 'aeroplane': 1, 'bicycle': 2, 'bird': 3, 'boat': 4,
-    'bottle': 5, 'bus': 6, 'car': 7, 'cat': 8, 'chair': 9, 'cow': 10,
-    'diningtable': 11, 'dog': 12, 'horse': 13, 'motorbike': 14, 'person': 15,
-    'pottedplant': 16, 'sheep': 17, 'sofa': 18, 'train': 19, 'tvmonitor': 20,
+    'background': 0,
+    'aeroplane': 1,
+    'bicycle': 2,
+    'bird': 3,
+    'boat': 4,
+    'bottle': 5,
+    'bus': 6,
+    'car': 7,
+    'cat': 8,
+    'chair': 9,
+    'cow': 10,
+    'diningtable': 11,
+    'dog': 12,
+    'horse': 13,
+    'motorbike': 14,
+    'person': 15,
+    'pottedplant': 16,
+    'sheep': 17,
+    'sofa': 18,
+    'train': 19,
+    'tvmonitor': 20,
 }
 
-
 GLOBAL_IMG_ID = 0  # global image id.
 GLOBAL_ANN_ID = 0  # global annotation id.
 
@@ -101,14 +116,14 @@ def dict_to_tf_example(data,
   by the raw data.
 
   Args:
-    data: dict holding PASCAL XML fields for a single image (obtained by
-      running tfrecord_util.recursive_parse_xml_to_dict)
+    data: dict holding PASCAL XML fields for a single image (obtained by running
+      tfrecord_util.recursive_parse_xml_to_dict)
     dataset_directory: Path to root directory holding PASCAL dataset
     label_map_dict: A map from string label names to integers ids.
     ignore_difficult_instances: Whether to skip difficult instances in the
       dataset  (default: False).
-    image_subdirectory: String specifying subdirectory within the
-      PASCAL dataset directory holding the actual image data.
+    image_subdirectory: String specifying subdirectory within the PASCAL dataset
+      directory holding the actual image data.
     ann_json_dict: annotation json dictionary.
 
   Returns:
@@ -184,26 +199,42 @@ def dict_to_tf_example(data,
         }
         ann_json_dict['annotations'].append(ann)
 
-  example = tf.train.Example(features=tf.train.Features(feature={
-      'image/height': tfrecord_util.int64_feature(height),
-      'image/width': tfrecord_util.int64_feature(width),
-      'image/filename': tfrecord_util.bytes_feature(
-          data['filename'].encode('utf8')),
-      'image/source_id': tfrecord_util.bytes_feature(
-          str(image_id).encode('utf8')),
-      'image/key/sha256': tfrecord_util.bytes_feature(key.encode('utf8')),
-      'image/encoded': tfrecord_util.bytes_feature(encoded_jpg),
-      'image/format': tfrecord_util.bytes_feature('jpeg'.encode('utf8')),
-      'image/object/bbox/xmin': tfrecord_util.float_list_feature(xmin),
-      'image/object/bbox/xmax': tfrecord_util.float_list_feature(xmax),
-      'image/object/bbox/ymin': tfrecord_util.float_list_feature(ymin),
-      'image/object/bbox/ymax': tfrecord_util.float_list_feature(ymax),
-      'image/object/class/text': tfrecord_util.bytes_list_feature(classes_text),
-      'image/object/class/label': tfrecord_util.int64_list_feature(classes),
-      'image/object/difficult': tfrecord_util.int64_list_feature(difficult_obj),
-      'image/object/truncated': tfrecord_util.int64_list_feature(truncated),
-      'image/object/view': tfrecord_util.bytes_list_feature(poses),
-  }))
+  example = tf.train.Example(
+      features=tf.train.Features(
+          feature={
+              'image/height':
+                  tfrecord_util.int64_feature(height),
+              'image/width':
+                  tfrecord_util.int64_feature(width),
+              'image/filename':
+                  tfrecord_util.bytes_feature(data['filename'].encode('utf8')),
+              'image/source_id':
+                  tfrecord_util.bytes_feature(str(image_id).encode('utf8')),
+              'image/key/sha256':
+                  tfrecord_util.bytes_feature(key.encode('utf8')),
+              'image/encoded':
+                  tfrecord_util.bytes_feature(encoded_jpg),
+              'image/format':
+                  tfrecord_util.bytes_feature('jpeg'.encode('utf8')),
+              'image/object/bbox/xmin':
+                  tfrecord_util.float_list_feature(xmin),
+              'image/object/bbox/xmax':
+                  tfrecord_util.float_list_feature(xmax),
+              'image/object/bbox/ymin':
+                  tfrecord_util.float_list_feature(ymin),
+              'image/object/bbox/ymax':
+                  tfrecord_util.float_list_feature(ymax),
+              'image/object/class/text':
+                  tfrecord_util.bytes_list_feature(classes_text),
+              'image/object/class/label':
+                  tfrecord_util.int64_list_feature(classes),
+              'image/object/difficult':
+                  tfrecord_util.int64_list_feature(difficult_obj),
+              'image/object/truncated':
+                  tfrecord_util.int64_list_feature(truncated),
+              'image/object/view':
+                  tfrecord_util.bytes_list_feature(poses),
+          }))
   return example
 
 
@@ -222,8 +253,8 @@ def main(_):
 
   logging.info('writing to output path: %s', FLAGS.output_path)
   writers = [
-      tf.python_io.TFRecordWriter(
-          FLAGS.output_path + '-%05d-of-%05d.tfrecord' % (i, FLAGS.num_shards))
+      tf.python_io.TFRecordWriter(FLAGS.output_path + '-%05d-of-%05d.tfrecord' %
+                                  (i, FLAGS.num_shards))
       for i in range(FLAGS.num_shards)
   ]
 
@@ -260,9 +291,12 @@ def main(_):
       xml = etree.fromstring(xml_str)
       data = tfrecord_util.recursive_parse_xml_to_dict(xml)['annotation']
 
-      tf_example = dict_to_tf_example(data, FLAGS.data_dir, label_map_dict,
-                                      FLAGS.ignore_difficult_instances,
-                                      ann_json_dict=ann_json_dict)
+      tf_example = dict_to_tf_example(
+          data,
+          FLAGS.data_dir,
+          label_map_dict,
+          FLAGS.ignore_difficult_instances,
+          ann_json_dict=ann_json_dict)
       writers[idx % FLAGS.num_shards].write(tf_example.SerializeToString())
 
   for writer in writers:

efficientdet/g3doc/faq.md

@@ -7,20 +7,30 @@ go/g3doc-canonical-go-links
 freshness: { owner: 'tanmingxing' reviewed: '2020-05-09' }
 *-->
 
-
 [TOC]
 
 ## For Users
 
 ### How can I convert the saved model to tflite?
 
-Unfortunately, there is no way to do that with the current public tensorflow release due to some issues in tf converter.
-We have some internal fixes, which could potentially be available with the next TensorFlow release.
+Unfortunately, there is no way to do that with the current public tensorflow
+release due to some issues in tf converter. We have some internal fixes, which
+could potentially be available with the next TensorFlow release.
 
 ## For Developers
 
 ### How can I format my code for PRs?
 
+Please use [yapf](https://github.com/google/yapf) with option
+--style='{based_on_style: google, indent_width: 2}'. You can also save the
+following file to ~/.config/yapf/style:
+
+    [style]
+    based_on_style = google
+    indent_width = 2
+
+If you want to check the format with lint, please run:
+
     !pylint --rcfile=../.pylintrc your_file.py
 
 ### How can I run all tests?

efficientdet/inference.py

@@ -223,11 +223,9 @@ def det_post_process_combined(params, cls_outputs, box_outputs, scales,
       cls_outputs[level] = tf.transpose(cls_outputs[level], [0, 2, 3, 1])
       box_outputs[level] = tf.transpose(box_outputs[level], [0, 2, 3, 1])
 
-    cls_outputs_all.append(tf.reshape(
-        cls_outputs[level],
-        [batch_size, -1, params['num_classes']]))
-    box_outputs_all.append(tf.reshape(
-        box_outputs[level], [batch_size, -1, 4]))
+    cls_outputs_all.append(
+        tf.reshape(cls_outputs[level], [batch_size, -1, params['num_classes']]))
+    box_outputs_all.append(tf.reshape(box_outputs[level], [batch_size, -1, 4]))
   cls_outputs_all = tf.concat(cls_outputs_all, 1)
   box_outputs_all = tf.concat(box_outputs_all, 1)
 
@@ -261,11 +259,8 @@ def det_post_process_combined(params, cls_outputs, box_outputs, scales,
   ymax = tf.clip_by_value(nmsed_boxes[..., 2], 0, image_size[0]) * scales
   xmax = tf.clip_by_value(nmsed_boxes[..., 3], 0, image_size[1]) * scales
 
-  detection_list = [
-      # Format: (image_ids, ymin, xmin, ymax, xmax, score, class)
-      image_ids, ymin, xmin, ymax, xmax, nmsed_scores,
-      tf.cast(nmsed_classes + 1, tf.float32)
-  ]
+  classes = tf.cast(nmsed_classes + 1, tf.float32)
+  detection_list = [image_ids, ymin, xmin, ymax, xmax, nmsed_scores, classes]
   detections = tf.stack(detection_list, axis=2, name='detections')
   return detections
 

efficientdet/iou_utils.py

@@ -169,8 +169,8 @@ def iou_loss(pred_boxes: FloatType,
 
   iou_loss_list = []
   for i in range(0, len(pred_boxes_list), 4):
-    pred_boxes = pred_boxes_list[i: i + 4]
-    target_boxes = target_boxes_list[i: i + 4]
+    pred_boxes = pred_boxes_list[i:i + 4]
+    target_boxes = target_boxes_list[i:i + 4]
 
     # Compute mask.
     t_ymin, t_xmin, t_ymax, t_xmax = target_boxes
@@ -183,4 +183,3 @@ def iou_loss(pred_boxes: FloatType,
   if len(iou_loss_list) == 1:
     return iou_loss_list[0]
   return tf.reduce_sum(tf.stack(iou_loss_list), 0)
-

efficientdet/model_inspect.py

@@ -36,7 +36,6 @@
 import utils
 from tensorflow.python.client import timeline  # pylint: disable=g-direct-tensorflow-import
 
-
 flags.DEFINE_string('model_name', 'efficientdet-d0', 'Model.')
 flags.DEFINE_string('logdir', '/tmp/deff/', 'log directory.')
 flags.DEFINE_string('runmode', 'dry', 'Run mode: {freeze, bm, dry}')
@@ -117,7 +116,8 @@ def __init__(self,
 
     self.model_config = model_config
 
-  def build_model(self, inputs: tf.Tensor,
+  def build_model(self,
+                  inputs: tf.Tensor,
                   is_training: bool = False) -> List[tf.Tensor]:
     """Build model with inputs and labels and print out model stats."""
     logging.info('start building model')
@@ -166,7 +166,7 @@ def saved_model_inference(self, image_path_pattern, output_dir, **kwargs):
     num_batches = (len(all_files) + batch_size - 1) // batch_size
 
     for i in range(num_batches):
-      batch_files = all_files[i * batch_size: (i + 1) * batch_size]
+      batch_files = all_files[i * batch_size:(i + 1) * batch_size]
       height, width = self.model_config.image_size
       images = [Image.open(f) for f in batch_files]
       if len(set([m.size for m in images])) > 1:
@@ -225,8 +225,9 @@ def saved_model_video(self, video_path: Text, output_video: Text, **kwargs):
     out_ptr = None
     if output_video:
       frame_width, frame_height = int(cap.get(3)), int(cap.get(4))
-      out_ptr = cv2.VideoWriter(output_video, cv2.VideoWriter_fourcc(
-          'm', 'p', '4', 'v'), 25, (frame_width, frame_height))
+      out_ptr = cv2.VideoWriter(output_video,
+                                cv2.VideoWriter_fourcc('m', 'p', '4', 'v'), 25,
+                                (frame_width, frame_height))
 
     while cap.isOpened():
       # Capture frame-by-frame
@@ -320,7 +321,10 @@ def freeze_model(self) -> Tuple[Text, Text]:
 
     return graphdef
 
-  def benchmark_model(self, warmup_runs, bm_runs, num_threads,
+  def benchmark_model(self,
+                      warmup_runs,
+                      bm_runs,
+                      num_threads,
                       trace_filename=None):
     """Benchmark model."""
     if self.tensorrt:
@@ -385,16 +389,18 @@ def benchmark_model(self, warmup_runs, bm_runs, num_threads,
         run_options = tf.RunOptions()
         run_options.trace_level = tf.RunOptions.FULL_TRACE
         run_metadata = tf.RunMetadata()
-        sess.run(output_name, feed_dict={input_name: img},
-                 options=run_options, run_metadata=run_metadata)
+        sess.run(
+            output_name,
+            feed_dict={input_name: img},
+            options=run_options,
+            run_metadata=run_metadata)
         logging.info('Dumping trace to %s', trace_filename)
         trace_dir = os.path.dirname(trace_filename)
         if not tf.io.gfile.exists(trace_dir):
           tf.io.gfile.makedirs(trace_dir)
         with tf.io.gfile.GFile(trace_filename, 'w') as trace_file:
           trace = timeline.Timeline(step_stats=run_metadata.step_stats)
-          trace_file.write(
-              trace.generate_chrome_trace_format(show_memory=True))
+          trace_file.write(trace.generate_chrome_trace_format(show_memory=True))
 
   def convert_tr(self, graph_def, fetches):
     """Convert to TensorRT."""
@@ -432,18 +438,20 @@ def run_model(self, runmode, **kwargs):
       if runmode == 'saved_model':
         self.export_saved_model(**config_dict)
       elif runmode == 'infer':
-        self.inference_single_image(
-            kwargs['input_image'], kwargs['output_image_dir'], **config_dict)
+        self.inference_single_image(kwargs['input_image'],
+                                    kwargs['output_image_dir'], **config_dict)
       elif runmode == 'saved_model_infer':
-        self.saved_model_inference(
-            kwargs['input_image'], kwargs['output_image_dir'], **config_dict)
+        self.saved_model_inference(kwargs['input_image'],
+                                   kwargs['output_image_dir'], **config_dict)
       elif runmode == 'saved_model_video':
-        self.saved_model_video(
-            kwargs['input_video'], kwargs['output_video'], **config_dict)
+        self.saved_model_video(kwargs['input_video'], kwargs['output_video'],
+                               **config_dict)
     elif runmode == 'bm':
-      self.benchmark_model(warmup_runs=5, bm_runs=kwargs.get('bm_runs', 10),
-                           num_threads=kwargs.get('threads', 0),
-                           trace_filename=kwargs.get('trace_filename', None))
+      self.benchmark_model(
+          warmup_runs=5,
+          bm_runs=kwargs.get('bm_runs', 10),
+          num_threads=kwargs.get('threads', 0),
+          trace_filename=kwargs.get('trace_filename', None))
     else:
       raise ValueError('Unkown runmode {}'.format(runmode))
 

efficientdet/model_inspect_test.py

@@ -172,8 +172,8 @@ def test_saved_model_graph_infer(self):
     tf.reset_default_graph()
 
     # Use the frozen graph to do inference.
-    inspector.saved_model_dir = os.path.join(
-        self.params['saved_model_dir'], 'efficientdet-d0_frozen.pb')
+    inspector.saved_model_dir = os.path.join(self.params['saved_model_dir'],
+                                             'efficientdet-d0_frozen.pb')
     outdir = os.path.join(self.tempdir, 'pb_infer_imgout')
     os.mkdir(outdir)