convert 'None' and '' string values to NoneType before forwarding them to fluentd

Author: T0MASD

fluent/handler.py

@@ -120,10 +120,10 @@ def _structuring(self, data, record):
             self._add_dic(data, {'message': msg})
 
     @staticmethod
-    def _add_dic(data, dic):
+    def _add_dic(data, dic, nonevalues=['None','']):
         for key, value in dic.items():
             if isinstance(key, basestring):
-                data[str(key)] = value
+                data[str(key)] = None if value in nonevalues else value
 
 
 class FluentHandler(logging.Handler):

tests/test_handler.py

@@ -151,6 +151,28 @@ def test_custom_field_fill_missing_fmt_key_is_true(self):
         # field defaults to none if not in log record
         self.assertIsNone(data[0][2]['custom_field'])
 
+    def test_custom_field_convert_none_strings(self):
+        handler = fluent.handler.FluentHandler('app.follow', port=self._port)
+
+        logging.basicConfig(level=logging.INFO)
+        log = logging.getLogger('fluent.test')
+        handler.setFormatter(
+            fluent.handler.FluentRecordFormatter(fmt={
+                'name': '%(name)s',
+                }
+            )
+        )
+        log.addHandler(handler)
+        log.info({'name': 'None', 'sample':''})
+        log.removeHandler(handler)
+        handler.close()
+
+        data = self.get_data()
+        # field should be none
+        self.assertIsNone(data[0][2]['name'])
+        self.assertIsNone(data[0][2]['sample'])
+
+
     def test_json_encoded_message(self):
         handler = fluent.handler.FluentHandler('app.follow', port=self._port)