Skip to content

Commit 8c6ba0d

Browse files
sigurdmsigurdm
committed
Image proxy
1 parent 412b3f2 commit 8c6ba0d

File tree

10 files changed

+931
-0
lines changed

10 files changed

+931
-0
lines changed

‎pkg/image_proxy/Dockerfile‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Use latest stable channel SDK.
2+
FROM dart:3.9.0 AS build
3+
4+
ARG token
5+
6+
ENV PUB_ENVIRONMENT="bot"
7+
ENV PUB_CACHE="/project/.pub-cache"
8+
9+
# Resolve app dependencies.
10+
WORKDIR /app
11+
COPY . .
12+
RUN dart pub get --enforce-lockfile
13+
RUN dart compile exe pkg/image_proxy/bin/server.dart -o server
14+
15+
# Build minimal serving image from AOT-compiled `/server`
16+
# and the pre-built AOT-runtime in the `/runtime/` directory of the base image.
17+
FROM scratch
18+
COPY --from=build /runtime/ /
19+
COPY --from=build /app/server /app/bin/
20+
RUN echo $token > token
21+
22+
# Start server.
23+
EXPOSE 8080
24+
CMD ["/app/bin/server"]

‎pkg/image_proxy/README.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Image proxy for pub.dev.
2+
3+
4+
Will forward requests to a url, when given a request like:
5+
```
6+
https://external-image.pub.dev/<base64(hmac(url,hmac_kms(date))>/<date>/<urlencode(url)>
7+
```
8+
9+
date is a "microsecond after epoch" timestamp of a specific date's midnight.
10+
11+
hmac_kms is calculated in KMS with the key version at HMAC_KEY_ID.
12+
13+
## Development
14+
15+
To build the docker image (from the repository root):
16+
17+
```
18+
docker build -t image-proxy-server . --file pkg/image_proxy/Dockerfile
19+
```
20+

‎pkg/image_proxy/bin/server.dart‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
export 'package:pub_dev_image_proxy/image_proxy_service.dart' show main;

‎pkg/image_proxy/lib/image_proxy_service.dart‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'dart:async';
6+
import 'dart:convert';
7+
import 'dart:io';
8+
import 'dart:typed_data';
9+
import 'package:crypto/crypto.dart';
10+
import 'package:googleapis_auth/googleapis_auth.dart';
11+
import 'package:http/http.dart' as http;
12+
13+
import 'package:retry/retry.dart';
14+
import 'package:shelf/shelf.dart' as shelf;
15+
import 'package:shelf/shelf_io.dart';
16+
import 'package:googleapis/cloudkms/v1.dart' as kms;
17+
import 'package:googleapis_auth/auth_io.dart' as auth;
18+
19+
class Config {
20+
final Uri secretsUrl;
21+
final int maxFileSize;
22+
Config({required this.secretsUrl, required this.maxFileSize});
23+
}
24+
25+
bool isTesting = Platform.environment['IMAGE_PROXY_TESTING'] == 'true';
26+
27+
/// The keys we currently allow the url to be signed with.
28+
Map<int, Uint8List> allowedKeys = {};
29+
30+
/// Ensure that [allowedKeys] contains keys for today and the two surrounding
31+
/// days.
32+
Future<void> updateAllowedKeys() async {
33+
final now = DateTime.now();
34+
final yesterday = DateTime(now.year, now.month, now.day - 1);
35+
final today = DateTime(now.year, now.month, now.day);
36+
final tomorrow = DateTime(now.year, now.month, now.day + 1);
37+
38+
for (final d in [yesterday, today, tomorrow]) {
39+
if (!allowedKeys.containsKey(d.millisecondsSinceEpoch)) {
40+
allowedKeys[d.millisecondsSinceEpoch] = isTesting
41+
? await getDailySecretMock(d)
42+
: await getDailySecret(d);
43+
print('Generating new key for ${d.toIso8601String()}');
44+
}
45+
}
46+
while (allowedKeys.length > 3) {
47+
final dates = allowedKeys.keys.toList()..sort();
48+
allowedKeys.remove(dates.first);
49+
}
50+
assert(allowedKeys.length == 3);
51+
}
52+
53+
late auth.AuthClient? _apiClient;
54+
55+
/// The client used for communicating with the google apis.
56+
Future<AuthClient> authClient() async {
57+
return (_apiClient ??= await retry(() async {
58+
return await auth.clientViaApplicationDefaultCredentials(scopes: []);
59+
}))!;
60+
}
61+
62+
Future<Uint8List> getDailySecretMock(DateTime day) async {
63+
return hmacSign(
64+
utf8.encode('fake secret'),
65+
utf8.encode(
66+
DateTime(day.year, day.month, day.day).toUtc().toIso8601String(),
67+
),
68+
);
69+
}
70+
71+
/// Requests a derived hmac key corresponding to [day] using.
72+
Future<Uint8List> getDailySecret(DateTime day) async {
73+
final api = kms.CloudKMSApi(await authClient());
74+
final response = await api
75+
.projects
76+
.locations
77+
.keyRings
78+
.cryptoKeys
79+
.cryptoKeyVersions
80+
.macSign(
81+
kms.MacSignRequest()
82+
..dataAsBytes = utf8.encode(
83+
DateTime(day.year, day.month, day.day).toUtc().toIso8601String(),
84+
),
85+
Platform.environment['HMAC_KEY_ID']!,
86+
);
87+
return response.macAsBytes as Uint8List;
88+
}
89+
90+
bool _constantTimeEquals(Uint8List a, Uint8List b) {
91+
if (a.length != b.length) return false;
92+
bool answer = true;
93+
for (var i = 0; i < a.length; i++) {
94+
answer &= a[i] == b[i];
95+
}
96+
return answer;
97+
}
98+
99+
// The client used for requesting the images.
100+
// Using raw dart:io client such that we can disable autoUncompress.
101+
final HttpClient client = HttpClient()..autoUncompress = false;
102+
103+
final maxImageSize = 1024 * 1024 * 10; // At most 10 MB.
104+
105+
Future<shelf.Response> handler(shelf.Request request) async {
106+
try {
107+
if (request.method != 'GET') {
108+
return shelf.Response.notFound('Unsupported method');
109+
}
110+
final segments = request.url.pathSegments;
111+
if (segments.length != 3) {
112+
return shelf.Response.badRequest(
113+
body:
114+
'malformed request, ${segments.length} should be of the form <base64(hmac(url,daily_secret))>/<date>/<urlencode(url)>',
115+
);
116+
}
117+
final Uint8List signature;
118+
try {
119+
signature = base64Decode(segments[0]);
120+
} on FormatException catch (_) {
121+
return shelf.Response.badRequest(
122+
body: 'malformed request, could not decode mac signature',
123+
);
124+
}
125+
final date = int.tryParse(segments[1]);
126+
if (date == null) {
127+
return shelf.Response.badRequest(body: 'malformed request, missing date');
128+
}
129+
final secret = allowedKeys[date];
130+
if (secret == null) {
131+
return shelf.Response.badRequest(
132+
body: 'malformed request, proxy url expired',
133+
);
134+
}
135+
136+
final imageUrl = segments[2];
137+
if (imageUrl.length > 1024) {
138+
return shelf.Response.badRequest(body: 'proxied url too long');
139+
}
140+
final imageUrlBytes = utf8.encode(imageUrl);
141+
142+
if (_constantTimeEquals(hmacSign(secret, imageUrlBytes), signature)) {
143+
final Uri parsedImageUrl;
144+
try {
145+
parsedImageUrl = Uri.parse(imageUrl);
146+
} on FormatException catch (e) {
147+
return shelf.Response.badRequest(body: 'Malformed proxied url $e');
148+
}
149+
if (!(parsedImageUrl.isScheme('http') ||
150+
parsedImageUrl.isScheme('https'))) {
151+
return shelf.Response.badRequest(
152+
body: 'Can only proxy http and https urls',
153+
);
154+
}
155+
if (!parsedImageUrl.isAbsolute) {
156+
return shelf.Response.badRequest(body: 'Can only proxy absolute urls');
157+
}
158+
159+
int statusCode;
160+
List<int> bytes;
161+
String? contentType;
162+
String? contentEncoding;
163+
try {
164+
(statusCode, bytes, contentType, contentEncoding) = await retry(
165+
maxDelay: isTesting ? Duration(seconds: 1) : Duration(seconds: 8),
166+
maxAttempts: isTesting ? 2 : 8,
167+
() async {
168+
final request = await client.getUrl(parsedImageUrl);
169+
request.headers.add('user-agent', 'pub-proxy');
170+
request.followRedirects = false;
171+
var response = await request.close();
172+
var redirectCount = 0;
173+
while (response.isRedirect) {
174+
await response.drain();
175+
redirectCount++;
176+
if (redirectCount > 10) {
177+
throw RedirectException('Too many redirects.');
178+
}
179+
final location = response.headers.value(
180+
HttpHeaders.locationHeader,
181+
);
182+
if (location == null) {
183+
throw RedirectException('No location header in redirect.');
184+
}
185+
final uri = parsedImageUrl.resolve(location);
186+
final request = await client.getUrl(uri);
187+
request.headers.add('user-agent', 'pub-proxy');
188+
// Set the body or headers as desired.
189+
request.followRedirects = false;
190+
response = await request.close();
191+
}
192+
switch (response.statusCode) {
193+
case final int statusCode && >= 500 && < 600:
194+
throw ServerSideException(statusCode: statusCode);
195+
case final int statusCode && >= 300 && < 400:
196+
throw ServerSideException(statusCode: statusCode);
197+
}
198+
final contentLength = response.contentLength;
199+
if (contentLength != -1 && contentLength > maxImageSize) {
200+
throw TooLargeException();
201+
}
202+
return (
203+
response.statusCode,
204+
await readAllBytes(
205+
response,
206+
contentLength == -1 ? maxImageSize : contentLength,
207+
),
208+
response.headers.value('content-type'),
209+
response.headers.value('content-encoding'),
210+
);
211+
},
212+
retryIf: (e) =>
213+
e is SocketException ||
214+
e is http.ClientException ||
215+
e is ServerSideException,
216+
);
217+
} on TooLargeException {
218+
return shelf.Response.badRequest(body: 'Image too large');
219+
} on RedirectException catch (e) {
220+
return shelf.Response.badRequest(body: e.message);
221+
} on ServerSideException catch (e) {
222+
return shelf.Response.badRequest(
223+
body: 'Failed to retrieve image. Status code ${e.statusCode}',
224+
);
225+
}
226+
227+
return shelf.Response(
228+
statusCode,
229+
body: bytes,
230+
headers: {
231+
'Cache-control': 'max-age=180, public',
232+
'content-type': ?contentType,
233+
'content-encoding': ?contentEncoding,
234+
},
235+
);
236+
} else {
237+
return shelf.Response.unauthorized('Bad hmac');
238+
}
239+
} catch (e, st) {
240+
stderr.writeln('Uncaught error: $e $st');
241+
rethrow;
242+
}
243+
}
244+
245+
void main(List<String> args) async {
246+
await updateAllowedKeys();
247+
Timer.periodic(Duration(hours: 1), (_) => updateAllowedKeys());
248+
final server = await serve(
249+
handler,
250+
InternetAddress.anyIPv6,
251+
int.tryParse(Platform.environment['IMAGE_PROXY_PORT'] ?? '') ?? 80,
252+
);
253+
print('Serving image proxy on ${server.address}:${server.port}');
254+
}
255+
256+
class TooLargeException implements Exception {
257+
TooLargeException();
258+
}
259+
260+
class ServerSideException implements Exception {
261+
int statusCode;
262+
ServerSideException({required this.statusCode});
263+
}
264+
265+
class RedirectException implements Exception {
266+
String message;
267+
RedirectException(this.message);
268+
}
269+
270+
Uint8List hmacSign(Uint8List key, Uint8List imageUrlBytes) {
271+
return Hmac(sha256, key).convert(imageUrlBytes).bytes as Uint8List;
272+
}
273+
274+
Future<Uint8List> readAllBytes(Stream<List<int>> stream, int maxBytes) async {
275+
final builder = BytesBuilder();
276+
277+
await for (final chunk in stream) {
278+
if (builder.length + chunk.length > maxBytes) {
279+
throw TooLargeException();
280+
}
281+
builder.add(chunk);
282+
}
283+
return builder.takeBytes();
284+
}

‎pkg/image_proxy/pubspec.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: pub_dev_image_proxy
2+
environment:
3+
sdk: ^3.9.0-0
4+
5+
resolution: workspace
6+
7+
dependencies:
8+
shelf: ^1.4.2
9+
crypto: ^3.0.6
10+
http: ^1.5.0
11+
retry: ^3.1.2
12+
gcloud: ^0.8.19
13+
googleapis_auth: ^2.0.0
14+
googleapis: ^14.0.0
15+
yaml: ^3.1.3
16+
dev_dependencies:
17+
test:
18+
lints:

0 commit comments

Comments
 (0)