Integrate remote cache server for Bazel

.github/workflows/ci.yml

@@ -13,13 +13,52 @@ jobs:
   setup-bazel:
     runs-on: ${{ matrix.os }}-latest
     strategy:
+      fail-fast: false
       matrix:
         os:
           - macos
           - ubuntu
           - windows
     steps:
-      - uses: actions/checkout@v4
-      - uses: ./
-      - if: failure() && runner.debug == '1'
+      - name: Checkout example
+        uses: actions/checkout@v4
+        with:
+          repository: bazelbuild/examples
+          sparse-checkout: frontend
+          sparse-checkout-cone-mode: false
+
+      - name: Checkout setup-bazel
+        uses: actions/checkout@v4
+        with:
+          path: setup-bazel
+
+      - name: Prepare environment (POSIX)
+        if: matrix.os != 'windows'
+        run: |
+          rm $(which bazel)
+          shopt -s dotglob
+          mv frontend/* .
+          rmdir frontend
+
+      - name: Prepare environment (Windows)
+        if: matrix.os == 'windows'
+        run: |
+          Remove-Item -Path (Get-Command bazel.exe).Source
+          Get-ChildItem -Path frontend -Recurse -File | Move-Item -Destination .
+          Remove-Item -Path frontend -Recurse -Force
+
+      - name: Setup Bazel
+        uses: ./setup-bazel/
+        with:
+          bazelisk-cache: true
+          bazelisk-version: 1.x
+          bazelrc: test --test_output=errors
+          remote-cache: true
+          repository-cache: true
+
+      - run: bazel build ...
+      - run: bazel test ...
+
+      - name: Start SSH session
+        if: failure() && runner.debug == '1'
         uses: mxschmitt/action-tmate@v3

action.yml

@@ -29,6 +29,10 @@ inputs:
     description: Google Cloud account key for remote cache
     required: false
     default: ""
+  remote-cache:
+    description: Use built-in remote cache server
+    required: false
+    default: "false"
   repository-cache:
     description: Cache repositories based on MODULE.bazel/WORKSPACE
     required: false

config.js

@@ -46,6 +46,16 @@ if (diskCacheEnabled) {
   }
 }
 
+const remoteCacheLogPath = core.toPosixPath(`${os.tmpdir()}/remote-cache-server.log`)
+const remoteCacheServerUrl = 'http://localhost:9889/cache'
+const remoteCacheEnabled = core.getBooleanInput('remote-cache')
+if (remoteCacheEnabled) {
+  bazelrc.push(`build --remote_cache=${remoteCacheServerUrl}`)
+  if (diskCacheEnabled) {
+    core.error('Disk cache and remote cache cannot be enabled at the same time')
+  }
+}
+
 const repositoryCacheConfig = core.getInput('repository-cache')
 const repositoryCacheEnabled = repositoryCacheConfig !== 'false'
 let repositoryCacheFiles = [
@@ -151,4 +161,9 @@ module.exports = {
     name: 'repository',
     paths: [bazelRepository]
   },
+  remoteCache: {
+    enabled: remoteCacheEnabled,
+    logPath: remoteCacheLogPath,
+    url: remoteCacheServerUrl,
+  }
 }

dist/main/index.js

@@ -52,6 +52,16 @@ if (diskCacheEnabled) {
   }
 }
 
+const remoteCacheLogPath = core.toPosixPath(`${os.tmpdir()}/remote-cache-server.log`)
+const remoteCacheServerUrl = 'http://localhost:9889/cache'
+const remoteCacheEnabled = core.getBooleanInput('remote-cache')
+if (remoteCacheEnabled) {
+  bazelrc.push(`build --remote_cache=${remoteCacheServerUrl}`)
+  if (diskCacheEnabled) {
+    core.error('Disk cache and remote cache cannot be enabled at the same time')
+  }
+}
+
 const repositoryCacheConfig = core.getInput('repository-cache')
 const repositoryCacheEnabled = repositoryCacheConfig !== 'false'
 let repositoryCacheFiles = [
@@ -157,6 +167,11 @@ module.exports = {
     name: 'repository',
     paths: [bazelRepository]
   },
+  remoteCache: {
+    enabled: remoteCacheEnabled,
+    logPath: remoteCacheLogPath,
+    url: remoteCacheServerUrl,
+  }
 }
 
 
@@ -96867,6 +96882,8 @@ const github = __nccwpck_require__(5438)
 const glob = __nccwpck_require__(8090)
 const tc = __nccwpck_require__(7784)
 const config = __nccwpck_require__(5532)
+const { spawn } = __nccwpck_require__(2081)
+const path = __nccwpck_require__(1017)
 
 async function run() {
   try {
@@ -96889,6 +96906,7 @@ async function setupBazel() {
   await restoreCache(config.diskCache)
   await restoreCache(config.repositoryCache)
   await restoreExternalCaches(config.externalCache)
+  await startRemoteCacheServer()
 }
 
 async function setupBazelisk() {
@@ -96899,7 +96917,7 @@ async function setupBazelisk() {
   core.startGroup('Setup Bazelisk')
   let toolPath = tc.find('bazelisk', config.bazeliskVersion)
   if (toolPath) {
-    core.debug(`Found in cache @ ${toolPath}`)
+    core.info(`Found in cache @ ${toolPath}`)
   } else {
     toolPath = await downloadBazelisk()
   }
@@ -96950,13 +96968,21 @@ async function downloadBazelisk() {
   }
 
   const url = asset.browser_download_url
-  core.debug(`Downloading from ${url}`)
+  core.info(`Downloading from ${url}`)
   const downloadPath = await tc.downloadTool(url, undefined, `token ${token}`)
 
-  core.debug('Adding to the cache...');
-  fs.chmodSync(downloadPath, '755');
-  const cachePath = await tc.cacheFile(downloadPath, 'bazel', 'bazelisk', version)
-  core.debug(`Successfully cached bazelisk to ${cachePath}`)
+  core.debug('Adding to the cache...')
+  fs.chmodSync(downloadPath, '755')
+
+  let bazelBinName = 'bazel'
+  let bazeliskBinName = 'bazelisk'
+  if (platform == 'windows') {
+    bazelBinName = `${bazelBinName}.exe`
+    bazeliskBinName = `${bazelBinName}.exe`
+  }
+
+  const cachePath = await tc.cacheFile(downloadPath, bazelBinName, bazeliskBinName, version)
+  core.info(`Successfully cached bazelisk to ${cachePath}`)
 
   return cachePath
 }
@@ -97035,6 +97061,28 @@ async function restoreCache(cacheConfig) {
   }())
 }
 
+async function startRemoteCacheServer() {
+  if (!config.remoteCache.enabled) {
+    return
+  }
+
+  core.startGroup("Start remote cache server")
+  core.info(`Remote cache server log file path: ${config.remoteCache.logPath}`)
+
+  const log = fs.openSync(config.remoteCache.logPath, 'a')
+  const remoteCacheServer = path.join(__dirname, '..', 'remote-cache-server', 'index.js')
+  const serverProcess = spawn(process.execPath, [remoteCacheServer], {
+    detached: true,
+    stdio: ['ignore', log, log]
+  })
+
+  core.info(`Started remote cache server with PID: ${serverProcess.pid}`)
+  core.saveState('remote-cache-server-pid', serverProcess.pid.toString())
+
+  serverProcess.unref()
+  core.endGroup()
+}
+
 run()
 
 })();

dist/main/remote-cache-server.js

@@ -0,0 +1,50 @@
+const http = require('http')
+const cache = require('@actions/cache')
+const fs = require('fs')
+
+// https://bazel.build/remote/caching#http-caching
+const server = http.createServer(async (req, res) => {
+  const { method, url } = req
+  const [, , cacheType, sha] = url.split('/')
+  const filePath = `/tmp/cache-${cacheType}-${sha}`
+
+  if (method === 'GET') {
+    try {
+      const cacheId = await cache.restoreCache([filePath], sha)
+      if (!cacheId) {
+        console.log(`Cache miss for ${sha}`)
+        res.writeHead(404)
+        return res.end('Cache miss')
+      }
+      const data = fs.readFileSync(filePath)
+      res.writeHead(200, { 'Content-Type': 'application/octet-stream' })
+      res.end(data)
+    } catch (error) {
+      console.error(`Error retrieving cache for ${sha}: ${error}`)
+      res.writeHead(500)
+      res.end('Internal Server Error')
+    }
+  } else if (method === 'PUT') {
+    const data = []
+    req.on('data', chunk => data.push(chunk))
+    req.on('end', async () => {
+      try {
+        fs.writeFileSync(filePath, Buffer.concat(data))
+        await cache.saveCache([filePath], sha)
+        console.log(`Cache saved for ${sha}`)
+        res.writeHead(201)
+        res.end('Cache saved')
+      } catch (error) {
+        console.error(`Error saving cache for ${sha}: ${error}`)
+        res.writeHead(500)
+        res.end('Internal Server Error')
+      }
+    })
+  } else {
+    res.writeHead(405)
+    res.end('Method Not Allowed')
+  }
+})
+
+const PORT = process.env.PORT || 8080
+server.listen(PORT, () => console.log(`Server listening on port ${PORT}`))

dist/post/index.js

@@ -52,6 +52,16 @@ if (diskCacheEnabled) {
   }
 }
 
+const remoteCacheLogPath = core.toPosixPath(`${os.tmpdir()}/remote-cache-server.log`)
+const remoteCacheServerUrl = 'http://localhost:9889/cache'
+const remoteCacheEnabled = core.getBooleanInput('remote-cache')
+if (remoteCacheEnabled) {
+  bazelrc.push(`build --remote_cache=${remoteCacheServerUrl}`)
+  if (diskCacheEnabled) {
+    core.error('Disk cache and remote cache cannot be enabled at the same time')
+  }
+}
+
 const repositoryCacheConfig = core.getInput('repository-cache')
 const repositoryCacheEnabled = repositoryCacheConfig !== 'false'
 let repositoryCacheFiles = [
@@ -157,6 +167,11 @@ module.exports = {
     name: 'repository',
     paths: [bazelRepository]
   },
+  remoteCache: {
+    enabled: remoteCacheEnabled,
+    logPath: remoteCacheLogPath,
+    url: remoteCacheServerUrl,
+  }
 }
 
 
@@ -95937,9 +95952,34 @@ const config = __nccwpck_require__(5532)
 const { getFolderSize } = __nccwpck_require__(4962)
 
 async function run() {
+  await stopRemoteCacheServer()
   await saveCaches()
 }
 
+async function stopRemoteCacheServer() {
+  if (!config.remoteCache.enabled) {
+    return
+  }
+
+  core.startGroup("Stop remote cache server")
+  const pid = core.getState('remote-cache-server-pid')
+  if (pid) {
+    try {
+      process.kill(pid, 'SIGTERM')
+      core.info(`Stopped remote cache server with PID: ${pid}`)
+    } catch (error) {
+      core.error(`Failed to stop remote cache server with PID: ${pid}. Error: ${error}`)
+    }
+  }
+
+  const logPath = config.remoteCache.logPath
+  if (fs.existsSync(logPath)) {
+    const logContent = fs.readFileSync(logPath, 'utf8')
+    core.debug(`Remote cache server log:\n${logContent}`)
+  }
+  core.endGroup()
+}
+
 async function saveCaches() {
   await saveCache(config.bazeliskCache)
   await saveCache(config.diskCache)