[DAPS-1847] - refactor: python package to be compatible with proto3 envelope (#1849)

* [DAPS-1848] - refactor: update authz files to use proto3. (#1851)
* [DAPS-1850] - refactor web server proto3 (#1852)

Author: JoshuaSBrown

CMakeLists.txt

@@ -128,8 +128,8 @@ endif()
 if( BUILD_WEB_SERVER )
   include(./cmake/Web.cmake)
 
-  file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/common/proto/common/" 
-     DESTINATION "${CMAKE_CURRENT_SOURCE_DIR}/web/proto/")
+  file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/common/proto3/common/" 
+     DESTINATION "${CMAKE_CURRENT_SOURCE_DIR}/web/proto3/")
 
   if( ENABLE_UNIT_TESTS )
     add_test(NAME unit_tests_web COMMAND "${DEPENDENCY_INSTALL_PATH}/nvm/versions/node/${LOCAL_NODE_VERSION}/bin/npm" "test")
@@ -199,7 +199,7 @@ endif()
 
 if( BUILD_PYTHON_CLIENT )
   # make target = pydatafed
-  file(COPY ${PROJECT_SOURCE_DIR}/external/DataFedDependencies/python/datafed_pkg/requirements.txt DESTINATION ${PROJECT_SOURCE_DIR}/python/datafed_pkg/requirements.txt)
+  file(COPY ${PROJECT_SOURCE_DIR}/external/DataFedDependencies/python/datafed_pkg/requirements.txt DESTINATION ${PROJECT_SOURCE_DIR}/python/datafed_pkg)
   add_subdirectory( python EXCLUDE_FROM_ALL )
 endif()
 
@@ -227,9 +227,9 @@ if( INSTALL_CORE_SERVER )
 endif()
 
 if( INSTALL_WEB_SERVER )
-  install( FILES ${ProtoFiles} DESTINATION ${DATAFED_INSTALL_PATH}/web )
   install( DIRECTORY ${PROJECT_SOURCE_DIR}/web/static DESTINATION ${DATAFED_INSTALL_PATH}/web )
   install( DIRECTORY ${PROJECT_SOURCE_DIR}/web/views DESTINATION ${DATAFED_INSTALL_PATH}/web )
+  install( DIRECTORY ${PROJECT_SOURCE_DIR}/web/proto3 DESTINATION ${DATAFED_INSTALL_PATH}/web )
   install( FILES ${PROJECT_SOURCE_DIR}/web/version.js DESTINATION ${DATAFED_INSTALL_PATH}/web )
 endif()
 

docker/Dockerfile.runtime

@@ -27,6 +27,7 @@ COPY ./scripts/dependency_versions.sh  ${BUILD_DIR}/scripts/
 RUN mkdir -p ${DATAFED_DIR}
 RUN mkdir -p /opt/datafed
 RUN mkdir -p /var/log/datafed
+RUN mkdir -p /opt/datafed/logs
 RUN chown -R datafed:root /opt/datafed
 RUN chown -R datafed:root /var/log/datafed
 RUN chown -R datafed:root ${DATAFED_DIR}

python/datafed_pkg/CMakeLists.txt

@@ -35,4 +35,4 @@ endforeach()
 add_subdirectory( datafed )
 
 add_custom_target( pydatafed )
-add_dependencies( pydatafed pydatafed_src)
+add_dependencies( pydatafed pydatafed_proto_src)

python/datafed_pkg/datafed/CLI.py

@@ -34,8 +34,6 @@
 from prompt_toolkit.history import FileHistory
 from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
 
-# from . import SDMS_Auth_pb2 as auth
-from . import Version_pb2
 from . import CommandLib
 from . import Config
 from . import VERSION
@@ -162,14 +160,14 @@ def run():
         except _NoCommand as e:
             # Be nice and switch to interactive when no command given
             if _interactive and _first:
-                api_version = f"{Version_pb2.DATAFED_COMMON_PROTOCOL_API_MAJOR}."
-                api_version += f"{Version_pb2.DATAFED_COMMON_PROTOCOL_API_MINOR}."
-                api_version += f"{Version_pb2.DATAFED_COMMON_PROTOCOL_API_PATCH}"
-                release_version = f"{Version_pb2.DATAFED_RELEASE_YEAR}."
-                release_version += f"{Version_pb2.DATAFED_RELEASE_MONTH}."
-                release_version += f"{Version_pb2.DATAFED_RELEASE_DAY}."
-                release_version += f"{Version_pb2.DATAFED_RELEASE_HOUR}."
-                release_version += f"{Version_pb2.DATAFED_RELEASE_MINUTE}"
+                api_version = f"{VERSION.DATAFED_COMMON_PROTOCOL_API_MAJOR}."
+                api_version += f"{VERSION.DATAFED_COMMON_PROTOCOL_API_MINOR}."
+                api_version += f"{VERSION.DATAFED_COMMON_PROTOCOL_API_PATCH}"
+                release_version = f"{VERSION.DATAFED_RELEASE_YEAR}."
+                release_version += f"{VERSION.DATAFED_RELEASE_MONTH}."
+                release_version += f"{VERSION.DATAFED_RELEASE_DAY}."
+                release_version += f"{VERSION.DATAFED_RELEASE_HOUR}."
+                release_version += f"{VERSION.DATAFED_RELEASE_MINUTE}"
                 _print_msg(1, f"Welcome to DataFed CLI, version {VERSION.__version__}")
                 _print_msg(
                     1, "               Release, version {}".format(release_version)

python/datafed_pkg/datafed/CMakeLists.txt

@@ -1,42 +1,60 @@
-cmake_minimum_required (VERSION 3.17.0)
+cmake_minimum_required(VERSION 3.17.0)
 
 # Copy py source to build package source dir
-file( GLOB SrcFiles ${CMAKE_CURRENT_SOURCE_DIR}/*.py )
+file(GLOB SrcFiles ${CMAKE_CURRENT_SOURCE_DIR}/*.py)
 foreach(file ${SrcFiles})
-    configure_file(${file} ${CMAKE_CURRENT_BINARY_DIR} COPYONLY )
+    configure_file(${file} ${CMAKE_CURRENT_BINARY_DIR} COPYONLY)
 endforeach()
 
-# Collect top-level proto files as dependencies
-file( GLOB ProtoFiles ${DataFed_SOURCE_DIR}/common/proto/common/*.proto )
+# Collect proto files from the new 1-1-1 directory structure
+file(GLOB_RECURSE ProtoFiles ${DataFed_SOURCE_DIR}/common/proto3/common/*.proto)
 
 # OBJECT - is needed because we don't want to compile to a binary
 # because we are dealing with python
 add_library(protobuf-target-py OBJECT ${ProtoFiles})
+
 protobuf_generate(
     LANGUAGE python
     TARGET protobuf-target-py
-    IMPORT_DIRS "${DataFed_SOURCE_DIR}/common/proto/common"
+    IMPORT_DIRS "${DataFed_SOURCE_DIR}/common/proto3/common"
     OUT_VAR protobuf-generated-files-py
     PROTOC_OUT_DIR "${CMAKE_CURRENT_BINARY_DIR}"
-  )
-
-add_custom_target( pydatafed_src DEPENDS protobuf-target-py )
-
-# By default this will output the proto py files in the CMAKE BINARY DIR
-add_custom_command( TARGET pydatafed_src POST_BUILD
-    COMMAND sed -i -r 's:^import.*_pb2:from . \\0:' ${protobuf-generated-files-py}
-    COMMAND ${DataFed_SOURCE_DIR}/python/pyproto_add_msg_idx.py ${DataFed_SOURCE_DIR}/common/proto/common/SDMS_Anon.proto ${CMAKE_CURRENT_BINARY_DIR}/SDMS_Anon_pb2.py
-    COMMAND ${DataFed_SOURCE_DIR}/python/pyproto_add_msg_idx.py ${DataFed_SOURCE_DIR}/common/proto/common/SDMS_Auth.proto ${CMAKE_CURRENT_BINARY_DIR}/SDMS_Auth_pb2.py 
-)     
-      
-# Crea#te copies of the files so they show up in the source folder as well 
-# for the purpose of testing
-add_custom_target( pydatafed_proto_src DEPENDS pydatafed_src )
-add_custom_command( TARGET pydatafed_proto_src POST_BUILD pydatafed_src 
-  COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/SDMS_Auth_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ 
-  COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/SDMS_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ 
-  COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/Version_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ 
-  COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/SDMS_Anon_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/
 )
 
+add_custom_target(pydatafed_src DEPENDS protobuf-target-py)
+
+# Proto subdirectories that protoc generates imports for
+set(PROTO_SUBDIRS anon auth enums messages)
+
+# Fix imports in generated pb2 files to use relative imports within the package.
+# protoc generates absolute imports like:
+#   from anon import ack_reply_pb2 as ...
+#   from enums import error_code_pb2 as ...
+#   import envelope_pb2 as ...
+# These must become relative imports:
+#   from .anon import ack_reply_pb2 as ...
+#   from .enums import error_code_pb2 as ...
+#   from . import envelope_pb2 as ...
 
+# Create the import fixup script
+add_custom_command(TARGET pydatafed_src POST_BUILD
+	COMMAND sh ${DataFed_SOURCE_DIR}/python/datafed_pkg/scripts/fix_proto_imports.sh  ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Rewriting protobuf imports to relative"
+)
+
+# Copy generated files back to source tree for testing
+add_custom_target(pydatafed_proto_src DEPENDS pydatafed_src)
+add_custom_command(TARGET pydatafed_proto_src POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy
+        ${CMAKE_CURRENT_BINARY_DIR}/envelope_pb2.py
+        ${CMAKE_CURRENT_SOURCE_DIR}/
+    # Copy subdirectories back to source for testing
+    COMMAND ${CMAKE_COMMAND} -E copy_directory
+        ${CMAKE_CURRENT_BINARY_DIR}/anon ${CMAKE_CURRENT_SOURCE_DIR}/anon
+    COMMAND ${CMAKE_COMMAND} -E copy_directory
+        ${CMAKE_CURRENT_BINARY_DIR}/auth ${CMAKE_CURRENT_SOURCE_DIR}/auth
+    COMMAND ${CMAKE_COMMAND} -E copy_directory
+        ${CMAKE_CURRENT_BINARY_DIR}/enums ${CMAKE_CURRENT_SOURCE_DIR}/enums
+    COMMAND ${CMAKE_COMMAND} -E copy_directory
+        ${CMAKE_CURRENT_BINARY_DIR}/messages ${CMAKE_CURRENT_SOURCE_DIR}/messages
+)