Skip to content

Commit e82adf4

Browse files
authored
Fly workers (#153)
1 parent d56a922 commit e82adf4

File tree

14 files changed

+130
-35
lines changed

14 files changed

+130
-35
lines changed

.docker/Dockerfile.roomote-worker

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# docker compose build roomote-worker && docker compose up roomote-worker
1+
# docker compose build --build-arg GH_TOKEN=$(npx dotenvx get GH_TOKEN -f .env.development) roomote-worker
22

33
FROM node:20-slim AS base
44

@@ -44,6 +44,18 @@ RUN wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor
4444
&& apt update && apt install -y code \
4545
&& rm -rf /var/lib/apt/lists/*
4646

47+
# Install ssh server
48+
# RUN apt-get update \
49+
# && apt-get install -y openssh-server \
50+
# && cp /etc/ssh/sshd_config /etc/ssh/sshd_config-original \
51+
# && sed -i 's/^#\s*Port.*/Port 2222/' /etc/ssh/sshd_config \
52+
# && sed -i 's/^#\s*PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config \
53+
# && mkdir -p /root/.ssh \
54+
# && chmod 700 /root/.ssh \
55+
# && mkdir /var/run/sshd \
56+
# && chmod 755 /var/run/sshd \
57+
# && rm -rf /var/lib/apt/lists /var/cache/apt/archives
58+
4759
WORKDIR /roo
4860

4961
# Install extensions
@@ -56,16 +68,22 @@ RUN mkdir -p /roo/.vscode \
5668
# Clone repo (requires $GH_TOKEN)
5769
ARG GH_TOKEN
5870
ENV GH_TOKEN=${GH_TOKEN}
59-
WORKDIR /roo/repos
6071
RUN git config --global user.email "[email protected]"
6172
RUN git config --global user.name "Roo Code"
6273
RUN git config --global credential.helper store
6374
RUN echo "https://oauth2:${GH_TOKEN}@github.com" > ~/.git-credentials
75+
76+
WORKDIR /roo/repos
6477
RUN gh repo clone RooCodeInc/Roo-Code
6578
WORKDIR /roo/repos/Roo-Code
6679
RUN gh repo set-default RooCodeInc/Roo-Code
6780
RUN pnpm install
6881

82+
WORKDIR /roo/repos
83+
RUN gh repo clone RooCodeInc/Roo-Code-Cloud
84+
WORKDIR /roo/repos/Roo-Code-Cloud
85+
RUN pnpm install
86+
6987
WORKDIR /roo
7088

7189
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml turbo.json .env.* ./
@@ -87,6 +105,10 @@ COPY apps/roomote ./apps/roomote/
87105

88106
WORKDIR /roo/apps/roomote
89107

108+
# COPY .docker/entrypoints/worker.sh /usr/local/bin/worker.sh
109+
# RUN chmod +x /usr/local/bin/worker.sh
110+
90111
ENV NODE_ENV=production
91112
ENV HOST_EXECUTION_METHOD=docker
92-
CMD ["sh", "-c", "npx dotenvx run -f ../../.env.${APP_ENV:-development} -- tsx src/lib/controller.ts"]
113+
# ENTRYPOINT ["/usr/local/bin/worker.sh"]
114+
CMD ["sleep", "infinity"]

.docker/entrypoints/worker.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash -e
2+
3+
echo "$AUTHORIZED_KEYS" > /root/.ssh/authorized_keys
4+
/usr/sbin/sshd
5+
6+
exec "$@"

.env.development

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ GH_TOKEN=encrypted:BGQvKArqcFk+y2UmBmzUZVBo6cNtDH20i0kHiqoGDRqAoFUJkc2hEumU6ukK+
1818

1919
OPENROUTER_API_KEY=encrypted:BA3bYeSEXFA0jnw3lmzLKz53ekkGr0K6sDLN6+nFGsa0TdJrYoKI5IcRPCMWuXIoLR1dlRi61eziWMQt0e2MgLU8zcR+zKif/2twlIzFlTU7OtFSwC1DPknxAG0pf3e2LSTDiLU4hRKgYbdJn8coA3rf/YY9NRVlmJV1gDLb9KVL89IWgT+fEn3Tzh383dSolr1ec/9RyAYVGpIYTg4O57OzQ9TaZp6InWs=
2020

21-
SLACK_API_TOKEN=encrypted:BI5T2ZFY35FaOfSzPT8qQyUdhJs+fmL6wOFp8cVd4hM0xcSJ8va+vk1CCpltNrk25uXCd8D+fQnITb3G+L/1z1X8eqZVobqRwxO39wTcqVv1d9AQSLxAmYcEoAEWowiXQei2qS+AMcBSjZhDfURbdOfYo1+XHj0NdfspvsB1YwC+3CN3iZC/TSVBkzcQDy82Db541HnfFxhIQg==
21+
SLACK_API_TOKEN=encrypted:BJyTFw3gugKZr6uaNZaqCH0UpW/mP4S8P1oL5NzeCvoZzSMJEcDrKsW+b+3Ojp0EuqBFI7T8SPUfRMr6ZTSI880+pKYWkmfAAl2jYYE5aw48LGkg+WJTjIW8g2jTEc3PXPYeMzszrJ2Igf8NuyDMLmhI9S9ew5OfhxD6bKwr73WdUKVDJ+MMVBQGIkbrt1PBX30kRuKZq5oqiA==
2222

2323

2424

.env.preview

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,5 @@ GH_TOKEN=encrypted:BMB+HR14pdhTojK7exBUDaWpiQGDWziDR5cdUMM/I24MpkdCtZrvHsf+uTc/g
1818

1919
OPENROUTER_API_KEY=encrypted:BDoXCMz3fV5DXa89cwd77/pZhw6tVIw96LvtyZEFwFQtQV1Qk+ZmtQoObiKZkfI4y1ZvpOSL2ivXfS7YY8QtLnl8uEG550FHq6s0FZsVdua+SlmoUhJOK0zV341GadCG3bc5aSMB5Wha6ml51CJzy/AKud4U4Zi3eXaj+nMxeXF8wtnHsw51JQ3F7N9VnYklyrb5FEin2SVovg3TPZk40saP8Q/YyDyU8e8=
2020

21-
SLACK_API_TOKEN=encrypted:BFmTkof1HoUg2UTgpm2wWdv5ZqVuldkgU5xLhGjgGFqRG7sOnAF5QdUVAfjV+890FbGF/BxBcb7bJsjMdf6+krq5U8p8Il2lucELdg1BoWisGcoy1+OCYoLxWm4RDrI/zovC09vaSnIdgIpz4T/4sNRbY5U8r7KaKXSCDPp2rCTUAC9Fc5cODlurbnMbfmPaFJIxKktqm5rfXg==
21+
SLACK_API_TOKEN=encrypted:BEmAqaFftmVNlZdx19KGXBDtJ1E1SFBJrAH52pwhFR00Eq6DJAOJSBLY+9lV4wsDRnhTaR8QBQ5fx4kX6751mqJ/Ldu9n7zYV+NTV5cWowvTh4jD+w7yufbUvNgnqs1CpvGXysaJoyQCQqQlUJ+wxlf+rb3yEN5CMR9y1Es/d+cXF2d73H2VfBDtVcKf1pCPn+kVeQW9XHquiw==
2222

.env.production

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,5 @@ GH_TOKEN=encrypted:BN93qO9v+VfEZFaIdRwyhwR2CPM0rw1QUE9SSm/qjIT6W9XnL4Gg6EStB9zfl
1818

1919
OPENROUTER_API_KEY=encrypted:BLBANC6AI/i5gHIeiBaY5lOogEEahxSLAvsxUpk6Z1FXtfrIEj/6EKA/CE1LzHOJEZNIXHTtlb41ONGQWuRFtgDcvbhSthZ5XZB4c9dcdPz0cxEU4RfWSRwjQxYZ8OMs1oLdr7NV8oR6IYwJ43lVfuePBYit7zX125Mp/25R7Kl50oSVREHnlfhtOMTOTQqLI7LutgGvQiDym1Ji1BFDaTBd4rTrkoymYxo=
2020

21-
SLACK_API_TOKEN=encrypted:BHrN/c76f9fFC6XlzfgRnrb2NXK1jlGrq5XbSqk0FM+t4BaLFGILW4RruQRnbBVVm4dGNVCP395H0rHXAQsFd25nPYbBUXiHY42Ap2jBfy3xVqm3dWhapffv4+MiTMdEJvpVY+yyxJwNAu2RFEEnI5I4UpogpY7yMFvHyzV2Jym3/9pJ1svQpCPqAzz79SXRLDQq8w1b7LmQhA==
21+
SLACK_API_TOKEN=encrypted:BHQC5Lv0k3VnZ0Ez4TcVvL0meJs8R5ysPFa47U3kkjOJstrtDa26F8Fjvd8/Qr7ztiKdJzGXpr9k4/Tm8mL9jZvG7HNH9D+fVHkyGZ2Mf7B97FdXjiDxA3f+Pifj0aTfM7U58tDrKTLeQ/6UzE3Emcxyg4CxKjAjEaRFOAiQxs1lVAtqzbeF7OCV6GHScpSe4zoAaFXvtQ07FA==
2222

apps/roomote/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
"worker": "dotenvx run -f ../../.env.development -- tsx src/lib/worker.ts",
1515
"worker:production": "dotenvx run -f ../../.env.production -- tsx src/lib/worker.ts",
1616
"cli": "dotenvx run -f ../../.env.development -- tsx src/lib/cli.ts",
17+
"cli:production": "dotenvx run -f ../../.env.production -- tsx src/lib/cli.ts",
1718
"clean": "rimraf .next .turbo"
1819
},
1920
"dependencies": {

apps/roomote/src/lib/controller.ts

Lines changed: 27 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import { spawn } from 'child_process';
2-
import fs from 'fs';
32
import { Queue } from 'bullmq';
43

54
import { redis } from './redis';
5+
import { isFlyMachine, isDockerContainer } from './utils';
66

77
export class WorkerController {
88
private readonly POLL_INTERVAL_MS = 5000;
@@ -75,22 +75,32 @@ export class WorkerController {
7575

7676
try {
7777
console.log(`Spawning worker: ${workerId}`);
78-
const isRunningInDocker = fs.existsSync('/.dockerenv');
79-
80-
const dockerArgs = [
81-
`--name roomote-${workerId}`,
82-
'--rm',
83-
'--network roo-code-cloud_default',
84-
`-e GH_TOKEN=${process.env.GH_TOKEN}`,
85-
'-v /var/run/docker.sock:/var/run/docker.sock',
86-
'-v /tmp/roomote:/var/log/roomote',
87-
];
88-
89-
const cliCommand = 'pnpm worker';
90-
91-
const command = isRunningInDocker
92-
? `docker run ${dockerArgs.join(' ')} roomote-worker sh -c "${cliCommand}"`
93-
: cliCommand;
78+
79+
const cliCommand =
80+
process.env.APP_ENV === 'production'
81+
? 'pnpm worker:production'
82+
: 'pnpm worker';
83+
84+
let command;
85+
86+
if (isFlyMachine()) {
87+
command = `fly machine run $(fly releases --image -a roomote-worker -j 2>/dev/null | jq -r '.[0].ImageRef') --vm-size performance-16x --rm --shell --command "pnpm worker:production" -a roomote-worker`;
88+
} else if (isDockerContainer()) {
89+
const dockerArgs = [
90+
`--name roomote-${workerId}`,
91+
'--rm',
92+
'--network roo-code-cloud_default',
93+
`-e APP_ENV=${process.env.APP_ENV || 'development'}`,
94+
`-e GH_TOKEN=${process.env.GH_TOKEN}`,
95+
`-e DOTENV_PRIVATE_KEY_PRODUCTION=${process.env.DOTENV_PRIVATE_KEY_PRODUCTION}`,
96+
'-v /var/run/docker.sock:/var/run/docker.sock',
97+
'-v /tmp/roomote:/var/log/roomote',
98+
];
99+
100+
command = `docker run ${dockerArgs.join(' ')} roomote-worker sh -c "${cliCommand}"`;
101+
} else {
102+
command = cliCommand;
103+
}
94104

95105
console.log('Spawning worker with command:', command);
96106

apps/roomote/src/lib/runTask.ts

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as path from 'path';
22
import * as os from 'node:os';
33
import * as crypto from 'node:crypto';
4+
import * as fs from 'node:fs';
45

56
import pWaitFor from 'p-wait-for';
67
import { execa } from 'execa';
@@ -17,7 +18,7 @@ import { IpcClient } from '@roo-code-cloud/ipc';
1718
import type { JobPayload, JobType } from '@roo-code-cloud/db';
1819

1920
import { Logger } from './logger';
20-
import { isDockerContainer } from './utils';
21+
import { isFlyMachine, isDockerContainer } from './utils';
2122
import { SlackNotifier } from './slack';
2223

2324
const TIMEOUT = 30 * 60 * 1_000;
@@ -71,26 +72,36 @@ export const runTask = async <T extends JobType>({
7172
`${crypto.randomUUID().slice(0, 8)}.sock`,
7273
);
7374

74-
const env = { ROO_CODE_IPC_SOCKET_PATH: ipcSocketPath };
7575
const controller = new AbortController();
7676
const cancelSignal = controller.signal;
77-
const containerized = isDockerContainer();
77+
const containerized = isFlyMachine() || isDockerContainer();
7878

7979
const codeCommand = containerized
80-
? `xvfb-run --auto-servernum --server-num=1 code --wait --log trace --disable-workspace-trust --disable-gpu --disable-lcd-text --no-sandbox --user-data-dir /roo/.vscode --password-store="basic" -n ${workspacePath}`
81-
: `code --disable-workspace-trust -n ${workspacePath}`;
80+
? `ROO_CODE_IPC_SOCKET_PATH=${ipcSocketPath} xvfb-run --auto-servernum --server-num=1 code --wait --log trace --disable-workspace-trust --disable-gpu --disable-lcd-text --no-sandbox --user-data-dir /roo/.vscode --password-store="basic" -n ${workspacePath}`
81+
: `ROO_CODE_IPC_SOCKET_PATH=${ipcSocketPath} code --disable-workspace-trust -n ${workspacePath}`;
8282

8383
logger.info(codeCommand);
8484

8585
const subprocess = execa({
86-
env,
8786
shell: '/bin/bash',
8887
cancelSignal,
8988
})`${codeCommand}`;
9089

9190
// If debugging, add `--verbose` to `command` and uncomment the following line.
9291
// subprocess.stdout.pipe(process.stdout)
9392

93+
try {
94+
await pWaitFor(() => fs.existsSync(ipcSocketPath), {
95+
interval: 250,
96+
timeout: 10_000,
97+
});
98+
} catch (_error) {
99+
logger.error(`IPC socket was not created within timeout: ${ipcSocketPath}`);
100+
throw new Error(
101+
`IPC socket was not created within timeout -> ${ipcSocketPath}`,
102+
);
103+
}
104+
94105
let client: IpcClient | undefined = undefined;
95106
let attempts = 5;
96107

apps/roomote/src/lib/slack.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,12 @@ export class SlackNotifier {
4747
this.logger?.error(
4848
`Slack API failed: ${response.status} ${response.statusText}`,
4949
);
50+
5051
return null;
5152
}
5253

5354
const result: SlackResponse = await response.json();
55+
console.log('🔗 Slack API Response ->', result);
5456

5557
if (!result.ok) {
5658
this.logger?.error(`Slack API error: ${result.error}`);
@@ -157,4 +159,8 @@ export class SlackNotifier {
157159
thread_ts: threadTs,
158160
});
159161
}
162+
163+
public async sendMessage(message: SlackMessage): Promise<string | null> {
164+
return await this.postMessage(message);
165+
}
160166
}

apps/roomote/src/lib/utils.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import * as fs from 'fs';
22
import * as path from 'path';
33

4+
export const isFlyMachine = () => {
5+
return !!process.env.FLY_IMAGE_REF;
6+
};
7+
48
export const isDockerContainer = () => {
59
try {
610
return fs.existsSync('/.dockerenv');

0 commit comments

Comments
 (0)