Merge pull request #185 from firebase/feature/fix-windows-vcpkg-rename-error

Feature/fix windows vcpkg rename error

Author: vimanyu

scripts/gha/build_desktop.py

@@ -65,7 +65,7 @@ def install_x86_support_libraries():
     utils.run_command(['apt', 'install', 'libsecret-1-dev:i386'], as_root=True)
 
 
-def install_cpp_dependencies_with_vcpkg(arch, msvc_runtime_library):
+def _install_cpp_dependencies_with_vcpkg(arch, msvc_runtime_library):
   """Install packages with vcpkg.
 
   This does the following,
@@ -99,14 +99,36 @@ def install_cpp_dependencies_with_vcpkg(arch, msvc_runtime_library):
   utils.run_command([vcpkg_executable_file_path, 'install',
                      '@' + vcpkg_response_file_path, '--disable-metrics'])
 
-  # Some errors in vcpkg installation are not bubbled up. Verify existence
-  # of certain important directories before proceeding.
-  utils.verify_vcpkg_build(vcpkg_triplet)
+def install_cpp_dependencies_with_vcpkg(arch, msvc_runtime_library, cleanup=True):
+  """Install packages with vcpkg and optionally cleanup any intermediates.
 
-  # Clear temporary directories and files created by vcpkg buildtrees
-  # could be several GBs and cause github runners to run out of space
-  utils.clean_vcpkg_temp_data()
+  This is a wrapper over a low level installation function and attempts the
+  installation twice, a second time after attempting to auto fix known issues.
 
+  Args:
+    arch (str): Architecture (eg: 'x86', 'x64').
+    msvc_runtime_library (str): Runtime library for MSVC (eg: 'static', 'dynamic').
+    cleanup (bool): Clean up intermediate files used during installation.
+
+  Raises:
+    (ValueError) If installation wasn't successful.
+  """
+  _install_cpp_dependencies_with_vcpkg(arch, msvc_runtime_library)
+  vcpkg_triplet = utils.get_vcpkg_triplet(arch, msvc_runtime_library)
+  # Verify the installation with an attempt to auto fix any issues.
+  success = utils.verify_vcpkg_build(vcpkg_triplet, attempt_auto_fix=True)
+  if not success:
+    print("Installation was not successful but auto fix was attempted. "
+          "Retrying installation...")
+    # Retry once more after attempted auto fix.
+    _install_cpp_dependencies_with_vcpkg(arch, msvc_runtime_library)
+    # Check for success again. If installation failed, this call will raise a ValueError.
+    success = utils.verify_vcpkg_build(vcpkg_triplet, attempt_auto_fix=False)
+
+  if cleanup:
+    # Clear temporary directories and files created by vcpkg buildtrees
+    # could be several GBs and cause github runners to run out of space
+    utils.clean_vcpkg_temp_data()
 
 def cmake_configure(build_dir, arch, msvc_runtime_library='static',
                     build_tests=True, config=None, target_format=None):
@@ -179,8 +201,13 @@ def main():
   if args.arch == 'x86' and utils.is_linux_os():
     install_x86_support_libraries()
 
-  # Install platform dependent cpp dependencies with vcpkg
-  install_cpp_dependencies_with_vcpkg(args.arch, args.msvc_runtime_library)
+  # Install C++ dependencies using vcpkg
+  install_cpp_dependencies_with_vcpkg(args.arch, args.msvc_runtime_library,
+                                      cleanup=True)
+
+  if args.vcpkg_step_only:
+    print("Exiting without building the Firebase C++ SDK as just vcpkg step was requested.")
+    return
 
   # CMake configure
   cmake_configure(args.build_dir, args.arch, args.msvc_runtime_library,
@@ -219,6 +246,7 @@ def parse_cmdline_args():
                       help='Runtime library for MSVC (static(/MT) or dynamic(/MD)')
   parser.add_argument('--build_dir', default='build', help='Output build directory')
   parser.add_argument('--build_tests', action='store_true', help='Build unit tests too')
+  parser.add_argument('--vcpkg_step_only', action='store_true', help='Just install cpp packages using vcpkg and exit.')
   parser.add_argument('--config', default='Release', help='Release/Debug config')
   parser.add_argument('--target', nargs='+', help='A list of CMake build targets (eg: firebase_app firebase_auth)')
   parser.add_argument('--target_format', default=None, help='(Mac only) whether to output frameworks (default) or libraries.')

scripts/gha/utils.py

@@ -169,15 +169,44 @@ def get_vcpkg_installation_script_path():
   return script_absolute_path
 
 
-def verify_vcpkg_build(vcpkg_triplet):
-  """Check if vcpkg installation finished successfully."""
+def verify_vcpkg_build(vcpkg_triplet, attempt_auto_fix=False):
+  """Check if vcpkg installation finished successfully.
+
+  Args:
+    vcpkg_triplet (str): Triplet name for vcpkg. Eg: 'x64-linux'
+    attempt_auto_fix (bool): If installation failed, try fixing some errors.
+
+  Returns:
+    (bool) True if everything looks good
+           False if installation failed but auto fix was attempted.
+                 Caller should retry installation in this case.
+  Raises:
+    (ValueError) Installation failed and auto fix was not attempted
+  """
   # At the very least, we should have an "installed" directory under vcpkg triplet.
   vcpkg_root_dir_path = get_vcpkg_root_path()
   installed_triplets_dir_path = os.path.join(vcpkg_root_dir_path, 'installed', vcpkg_triplet)
   if not os.path.exists(installed_triplets_dir_path):
+    if is_windows_os() and attempt_auto_fix:
+      # On some Windows machines with NFS drives, we have seen errors
+      # installing vcpkg due to permission issues while renaming temp directories.
+      # Manually renaming and re-running script makes it go through.
+      tools_dir_path = os.path.join(vcpkg_root_dir_path, 'downloads', 'tools')
+      for name in os.listdir(tools_dir_path):
+        # In the specific windows error that we noticed, the error occurs while
+        # trying to rename intermediate directories for donwloaded tools
+        # like "powershell.partial.<pid>" to "powershell". Renaming via python
+        # also runs into the same error. Workaround is to copy instead of rename.
+        if '.partial.' in name and os.path.isdir(os.path.join(tools_dir_path, name)):
+          expected_name = name.split('.partial.')[0]
+          shutil.copytree(os.path.join(tools_dir_path, name),
+                          os.path.join(tools_dir_path, expected_name))
+      return False
+
     raise ValueError("Could not find directory containing installed packages by vcpkg: {0}\n"
                      "Please check if there were errors during "
                      "vcpkg installation.".format(installed_triplets_dir_path))
+  return True
 
 
 def clean_vcpkg_temp_data():