refactor logging from multiline to formatted json (#154)

* refactor logging from multiline to formatted json

* regenerate api

* regenerate api

Author: rbiseck3

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.0.32-dev0
+
+* Improve logging of params to single line json
+
 ## 0.0.31
 
 * Support model name as api parameter

pipeline-notebooks/pipeline-general.ipynb

@@ -737,19 +737,22 @@
     "    file_content_type=None,\n",
     "    response_type=\"application/json\"\n",
     "):\n",
-    "    logger.debug(\n",
-    "        f\"\\npipeline_api input params:\\n\"\n",
-    "        f\"filename: {filename}\\n\"\n",
-    "        f\"m_strategy: {m_strategy}\\n\"\n",
-    "        f\"m_coordinates: {m_coordinates}\\n\"\n",
-    "        f\"m_ocr_languages: {m_ocr_languages}\\n\"\n",
-    "        f\"m_encoding: {m_encoding}\\n\"\n",
-    "        f\"m_xml_keep_tags: {m_xml_keep_tags}\\n\"\n",
-    "        f\"m_pdf_infer_table_structure: {m_pdf_infer_table_structure}\\n\"\n",
-    "        f\"m_hi_res_model_name: {m_hi_res_model_name}\\n\"\n",
-    "        f\"file_content_type: {file_content_type}\\n\"\n",
-    "        f\"response_type: {response_type}\"\n",
+    "    logger.debug(\"pipeline_api input params: {}\".format(\n",
+    "        json.dumps({\n",
+    "        \"request\": request,\n",
+    "        \"filename\": filename,\n",
+    "        \"m_strategy\": m_strategy,\n",
+    "        \"m_coordinates\": m_coordinates,\n",
+    "        \"m_ocr_languages\": m_ocr_languages,\n",
+    "        \"m_encoding\": m_encoding,\n",
+    "        \"m_xml_keep_tags\": m_xml_keep_tags,\n",
+    "        \"m_pdf_infer_table_structure\": m_pdf_infer_table_structure,\n",
+    "        \"m_hi_res_model_name\": m_hi_res_model_name,\n",
+    "        \"file_content_type\": file_content_type,\n",
+    "        \"response_type\": response_type\n",
+    "        }, default=str\n",
     "    )\n",
+    "    ))\n",
     "    if filename.endswith(\".msg\"):\n",
     "        # Note(yuming): convert file type for msg files\n",
     "        # since fast api might sent the wrong one.\n",
@@ -791,17 +794,18 @@
     "        pdf_infer_table_structure = False\n",
     "    \n",
     "    try:\n",
-    "        logger.debug(\n",
-    "            f\"\\npartition input data:\\n\"\n",
-    "            f\"content_type: {file_content_type}\\n\"\n",
-    "            f\"strategy: {strategy}\\n\"\n",
-    "            f\"ocr_languages: {ocr_languages}\\n\"\n",
-    "            f\"coordinates: {show_coordinates}\\n\"\n",
-    "            f\"pdf_infer_table_structure: {pdf_infer_table_structure}\\n\"\n",
-    "            f\"encoding: {encoding}\\n\"\n",
-    "            f\"model_name: {hi_res_model_name}\\n\"\n",
-    "            f\"xml_keep_tags: {xml_keep_tags}\\n\"\n",
-    "        )\n",
+    "        logger.debug(\"partition input data: {}\".format(\n",
+    "            json.dumps({\n",
+    "            \"content_type\": file_content_type,\n",
+    "            \"strategy\": strategy,\n",
+    "            \"ocr_languages\": ocr_languages,\n",
+    "            \"coordinates\": show_coordinates,\n",
+    "            \"pdf_infer_table_structure\": pdf_infer_table_structure,\n",
+    "            \"encoding\": encoding,\n",
+    "            \"model_name\": hi_res_model_name,\n",
+    "            \"xml_keep_tags\": xml_keep_tags\n",
+    "        }, default=str)\n",
+    "        ))\n",
     "          \n",
     "        if file_content_type == \"application/pdf\" and pdf_parallel_mode_enabled:\n",
     "            elements = partition_pdf_splits(\n",

prepline_general/api/general.py

@@ -210,17 +210,24 @@ def pipeline_api(
     response_type="application/json",
 ):
     logger.debug(
-        f"\npipeline_api input params:\n"
-        f"filename: {filename}\n"
-        f"m_strategy: {m_strategy}\n"
-        f"m_coordinates: {m_coordinates}\n"
-        f"m_ocr_languages: {m_ocr_languages}\n"
-        f"m_encoding: {m_encoding}\n"
-        f"m_xml_keep_tags: {m_xml_keep_tags}\n"
-        f"m_pdf_infer_table_structure: {m_pdf_infer_table_structure}\n"
-        f"m_hi_res_model_name: {m_hi_res_model_name}\n"
-        f"file_content_type: {file_content_type}\n"
-        f"response_type: {response_type}"
+        "pipeline_api input params: {}".format(
+            json.dumps(
+                {
+                    "request": request,
+                    "filename": filename,
+                    "m_strategy": m_strategy,
+                    "m_coordinates": m_coordinates,
+                    "m_ocr_languages": m_ocr_languages,
+                    "m_encoding": m_encoding,
+                    "m_xml_keep_tags": m_xml_keep_tags,
+                    "m_pdf_infer_table_structure": m_pdf_infer_table_structure,
+                    "m_hi_res_model_name": m_hi_res_model_name,
+                    "file_content_type": file_content_type,
+                    "response_type": response_type,
+                },
+                default=str,
+            )
+        )
     )
     if filename.endswith(".msg"):
         # Note(yuming): convert file type for msg files
@@ -266,15 +273,21 @@ def pipeline_api(
 
     try:
         logger.debug(
-            f"\npartition input data:\n"
-            f"content_type: {file_content_type}\n"
-            f"strategy: {strategy}\n"
-            f"ocr_languages: {ocr_languages}\n"
-            f"coordinates: {show_coordinates}\n"
-            f"pdf_infer_table_structure: {pdf_infer_table_structure}\n"
-            f"encoding: {encoding}\n"
-            f"model_name: {hi_res_model_name}\n"
-            f"xml_keep_tags: {xml_keep_tags}\n"
+            "partition input data: {}".format(
+                json.dumps(
+                    {
+                        "content_type": file_content_type,
+                        "strategy": strategy,
+                        "ocr_languages": ocr_languages,
+                        "coordinates": show_coordinates,
+                        "pdf_infer_table_structure": pdf_infer_table_structure,
+                        "encoding": encoding,
+                        "model_name": hi_res_model_name,
+                        "xml_keep_tags": xml_keep_tags,
+                    },
+                    default=str,
+                )
+            )
         )
 
         if file_content_type == "application/pdf" and pdf_parallel_mode_enabled:
@@ -445,7 +458,7 @@ def return_content_type(filename):
 
 
 @router.post("/general/v0/general")
-@router.post("/general/v0.0.31/general")
+@router.post("/general/v0.0.32/general")
 def pipeline_1(
     request: Request,
     gz_uncompressed_content_type: Optional[str] = Form(default=None),

preprocessing-pipeline-family.yaml

@@ -1,2 +1,2 @@
 name: general
-version: 0.0.31
+version: 0.0.32