When downloading external projects, provide a GitHub token if there is one. (#8899)

jonsimantov · web-flow · commit 76e2cd7187f6 · 2021-11-01T14:10:44.000-04:00
When downloading external projects, if the GITHUB_TOKEN environment variable is set, pass it to the external download cmake function via HTTP_HEADER.

This will prevent GitHub throttling the HTTP requests. Unauthenticated requests are limited by GitHub to 60 per IP address per hour, whereas authenticated requests are limited to 5,000 per user per hour.

(GitHub Actions runners automatically have the GITHUB_TOKEN environment variable set.)
diff --git a/cmake/external/GoogleUtilities.cmake b/cmake/external/GoogleUtilities.cmake
@@ -30,4 +30,5 @@ ExternalProject_Add(
   BUILD_COMMAND ""
   INSTALL_COMMAND ""
   TEST_COMMAND ""
+  HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"
 )
diff --git a/cmake/external/abseil-cpp.cmake b/cmake/external/abseil-cpp.cmake
@@ -30,4 +30,5 @@ ExternalProject_Add(
   BUILD_COMMAND ""
   INSTALL_COMMAND ""
   TEST_COMMAND ""
+  HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"
 )
diff --git a/cmake/external/benchmark.cmake b/cmake/external/benchmark.cmake
@@ -34,4 +34,5 @@ ExternalProject_Add(
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
+  HTTP_HEADER       "${EXTERNAL_PROJECT_HTTP_HEADER}"
 )
diff --git a/cmake/external/boringssl.cmake b/cmake/external/boringssl.cmake
@@ -37,4 +37,5 @@ ExternalProject_Add(
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
+  HTTP_HEADER       "${EXTERNAL_PROJECT_HTTP_HEADER}"
 )
diff --git a/cmake/external/c-ares.cmake b/cmake/external/c-ares.cmake
@@ -37,4 +37,5 @@ ExternalProject_Add(
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
+  HTTP_HEADER       "${EXTERNAL_PROJECT_HTTP_HEADER}"
 )
diff --git a/cmake/external/googletest.cmake b/cmake/external/googletest.cmake
@@ -34,4 +34,6 @@ ExternalProject_Add(
   BUILD_COMMAND ""
   INSTALL_COMMAND ""
   TEST_COMMAND ""
+
+  HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"
 )
diff --git a/cmake/external/grpc.cmake b/cmake/external/grpc.cmake
@@ -35,4 +35,6 @@ ExternalProject_Add(
   BUILD_COMMAND ""
   TEST_COMMAND ""
   INSTALL_COMMAND ""
+
+  HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"
 )
diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake
@@ -34,4 +34,6 @@ ExternalProject_Add(
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
+
+  HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"
 )
diff --git a/cmake/external/nanopb.cmake b/cmake/external/nanopb.cmake
@@ -34,4 +34,5 @@ ExternalProject_Add(
   BUILD_COMMAND ""
   INSTALL_COMMAND ""
   TEST_COMMAND ""
+  HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"
 )
diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake
@@ -37,4 +37,5 @@ ExternalProject_Add(
   BUILD_COMMAND ""
   INSTALL_COMMAND ""
   TEST_COMMAND ""
+  HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"
 )
diff --git a/cmake/external/zlib.cmake b/cmake/external/zlib.cmake
@@ -42,4 +42,5 @@ ExternalProject_Add(
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
+  HTTP_HEADER       "${EXTERNAL_PROJECT_HTTP_HEADER}"
 )
diff --git a/cmake/external_rules.cmake b/cmake/external_rules.cmake
@@ -18,6 +18,15 @@ function(download_external_sources)
   set(DOWNLOAD_BENCHMARK ${FIREBASE_IOS_BUILD_BENCHMARKS})
   set(DOWNLOAD_GOOGLETEST ${FIREBASE_IOS_BUILD_TESTS})
 
+  # If a GITHUB_TOKEN is present, use it for all external project downloads.
+  # This will prevent GitHub runners from being throttled by GitHub.
+  if(DEFINED ENV{GITHUB_TOKEN})
+    set(EXTERNAL_PROJECT_HTTP_HEADER "Authorization: token $ENV{GITHUB_TOKEN}")
+    message("Adding GITHUB_TOKEN header to ExternalProject downloads.")
+  else()
+    set(EXTERNAL_PROJECT_HTTP_HEADER "")
+  endif()
+
   execute_process(
     COMMAND
       ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}"
@@ -26,6 +35,7 @@ function(download_external_sources)
       -DFUZZING=${FUZZING}
       -DDOWNLOAD_BENCHMARK=${DOWNLOAD_BENCHMARK}
       -DDOWNLOAD_GOOGLETEST=${DOWNLOAD_GOOGLETEST}
+      -DEXTERNAL_PROJECT_HTTP_HEADER=${EXTERNAL_PROJECT_HTTP_HEADER}
       ${PROJECT_SOURCE_DIR}/cmake/external
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/external
   )

Original file line number	Diff line number	Diff line change
`@@ -30,4 +30,5 @@ ExternalProject_Add(`
`30`	`30`	`BUILD_COMMAND ""`
`31`	`31`	`INSTALL_COMMAND ""`
`32`	`32`	`TEST_COMMAND ""`
	`33`	`+ HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"`
`33`	`34`	`)`
Original file line number	Diff line number	Diff line change
`@@ -34,4 +34,5 @@ ExternalProject_Add(`
`34`	`34`	`BUILD_COMMAND ""`
`35`	`35`	`INSTALL_COMMAND ""`
`36`	`36`	`TEST_COMMAND ""`
	`37`	`+ HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"`
`37`	`38`	`)`
Original file line number	Diff line number	Diff line change
`@@ -37,4 +37,5 @@ ExternalProject_Add(`
`37`	`37`	`BUILD_COMMAND ""`
`38`	`38`	`INSTALL_COMMAND ""`
`39`	`39`	`TEST_COMMAND ""`
	`40`	`+ HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"`
`40`	`41`	`)`
Original file line number	Diff line number	Diff line change
`@@ -34,4 +34,6 @@ ExternalProject_Add(`
`34`	`34`	`BUILD_COMMAND ""`
`35`	`35`	`INSTALL_COMMAND ""`
`36`	`36`	`TEST_COMMAND ""`
	`37`	`+`
	`38`	`+ HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"`
`37`	`39`	`)`
Original file line number	Diff line number	Diff line change
`@@ -35,4 +35,6 @@ ExternalProject_Add(`
`35`	`35`	`BUILD_COMMAND ""`
`36`	`36`	`TEST_COMMAND ""`
`37`	`37`	`INSTALL_COMMAND ""`
	`38`	`+`
	`39`	`+ HTTP_HEADER "${EXTERNAL_PROJECT_HTTP_HEADER}"`
`38`	`40`	`)`