Skip to content

Commit 8b1d744

Browse files
fkorotkovfkorotkov
authored
Refactored proxyDownloadFromURLs to use omni-cache for URL proxying (#975)
* Refactored `proxyDownloadFromURLs` to use `omni-cache` for URL proxying Next step will be to release the CLI and replace streaming caching methods in CirrusCIService. * Update omni-cache dependency
1 parent ae12ef7 commit 8b1d744

File tree

5 files changed

+74
-56
lines changed

5 files changed

+74
-56
lines changed

‎go.mod‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ require (
5353
golang.org/x/crypto v0.45.0
5454
golang.org/x/sys v0.38.0
5555
golang.org/x/text v0.31.0
56-
google.golang.org/grpc v1.76.0
56+
google.golang.org/grpc v1.77.0
5757
google.golang.org/protobuf v1.36.10
5858
gopkg.in/natefinch/lumberjack.v2 v2.2.1
5959
gopkg.in/yaml.v3 v3.0.1
@@ -67,6 +67,7 @@ require (
6767
github.com/bmatcuk/doublestar v1.3.4
6868
github.com/cirruslabs/chacha v0.16.3
6969
github.com/cirruslabs/cirrus-ci-annotations v0.10.0
70+
github.com/cirruslabs/omni-cache v0.0.0-20251126154424-4794a1e10bc2
7071
github.com/cirruslabs/terminal v0.16.0
7172
github.com/docker/go-connections v0.6.0
7273
github.com/go-chi/render v1.0.3
@@ -114,6 +115,17 @@ require (
114115
github.com/ProtonMail/go-crypto v1.1.6 // indirect
115116
github.com/agext/levenshtein v1.2.3 // indirect
116117
github.com/ajg/form v1.5.1 // indirect
118+
github.com/aws/aws-sdk-go-v2 v1.39.6 // indirect
119+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect
120+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect
121+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect
122+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect
123+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect
124+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect
125+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 // indirect
126+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect
127+
github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0 // indirect
128+
github.com/aws/smithy-go v1.23.2 // indirect
117129
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
118130
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
119131
github.com/cloudflare/circl v1.6.1 // indirect
@@ -135,7 +147,7 @@ require (
135147
github.com/felixge/httpsnoop v1.0.4 // indirect
136148
github.com/fvbommel/sortorder v1.1.0 // indirect
137149
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
138-
github.com/go-jose/go-jose/v4 v4.1.2 // indirect
150+
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
139151
github.com/go-logr/logr v1.4.3 // indirect
140152
github.com/go-logr/stdr v1.2.2 // indirect
141153
github.com/go-ole/go-ole v1.3.0 // indirect
@@ -204,17 +216,18 @@ require (
204216
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
205217
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
206218
github.com/yusufpapurcu/wmi v1.2.4 // indirect
207-
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
219+
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
208220
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0 // indirect
209221
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
210222
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.35.0 // indirect
211223
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
212224
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0 // indirect
213-
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
225+
go.opentelemetry.io/proto/otlp v1.8.0 // indirect
214226
go.uber.org/multierr v1.11.0 // indirect
215227
golang.org/x/net v0.47.0 // indirect
216-
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
217-
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
228+
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect
229+
google.golang.org/genproto/googleapis/bytestream v0.0.0-20251111163417-95abcf5c77ba // indirect
230+
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect
218231
gopkg.in/warnings.v0 v0.1.2 // indirect
219232
gopkg.in/yaml.v2 v2.4.0 // indirect
220233
)

‎go.sum‎

Lines changed: 40 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,28 @@ github.com/avast/retry-go/v4 v4.7.0 h1:yjDs35SlGvKwRNSykujfjdMxMhMQQM0TnIjJaHB+Z
4747
github.com/avast/retry-go/v4 v4.7.0/go.mod h1:ZMPDa3sY2bKgpLtap9JRUgk2yTAba7cgiFhqxY2Sg6Q=
4848
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
4949
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
50+
github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+Xsqk=
51+
github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
52+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
53+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
54+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40=
55+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8=
56+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY=
57+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7awZqJzBAZ3PDD2zSfuP7w6P2knOzIn8M=
58+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA=
59+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k=
60+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
61+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
62+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4=
63+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4/go.mod h1:455WPHSwaGj2waRSpQp7TsnpOnBfw8iDfPfbwl7KPJE=
64+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM=
65+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg=
66+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ=
67+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8=
68+
github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0 h1:ef6gIJR+xv/JQWwpa5FYirzoQctfSJm7tuDe3SZsUf8=
69+
github.com/aws/aws-sdk-go-v2/service/s3 v1.90.0/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw=
70+
github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
71+
github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
5072
github.com/bartventer/httpcache v0.12.0 h1:GzOgiAD0fuM+PTmhxlsai3WJKr9dk6NBvjkuEZyLCpA=
5173
github.com/bartventer/httpcache v0.12.0/go.mod h1:78LO7c36hcyx2GQanbDozHTozNaIL32MT4N+lErHHFs=
5274
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -83,6 +105,10 @@ github.com/cirruslabs/echelon v1.9.0 h1:UtHAtoc+C7KZoYtbMCOL8JYA1Ndi6/4u0+gWxw9s
83105
github.com/cirruslabs/echelon v1.9.0/go.mod h1:Zk4IGe8upeefcwPGE143JCM7iwOkJnZ2ZFCbDWwoadA=
84106
github.com/cirruslabs/go-java-glob v0.1.0 h1:qC4Fzrq2sXKTLLsBb7ih7BNdCjCR6ZJAAK4nOZ6d/Ak=
85107
github.com/cirruslabs/go-java-glob v0.1.0/go.mod h1:+4vLmYqEnKtVAdeYsP5GCcPBcTuWsqi6P9bO6HraGFE=
108+
github.com/cirruslabs/omni-cache v0.0.0-20251126005918-e2a996654199 h1:CNIbDyxVJ0ic3nFH8i7pdAiGTNX1cqKu0hn05lk3JWk=
109+
github.com/cirruslabs/omni-cache v0.0.0-20251126005918-e2a996654199/go.mod h1:iiox2bLcisBBYXG8aUyvn6/UcuTrCexs+u4sv65qx6E=
110+
github.com/cirruslabs/omni-cache v0.0.0-20251126154424-4794a1e10bc2 h1:TBRvJzkJwHE+y6SZRrzmJuE8Q+Ig1D9VMBD/thX7RGc=
111+
github.com/cirruslabs/omni-cache v0.0.0-20251126154424-4794a1e10bc2/go.mod h1:iiox2bLcisBBYXG8aUyvn6/UcuTrCexs+u4sv65qx6E=
86112
github.com/cirruslabs/terminal v0.16.0 h1:RUHtBccwykpGI1pKb9aKqtMQnnz6DyzJZiQyI2e81zw=
87113
github.com/cirruslabs/terminal v0.16.0/go.mod h1:NYC5TZrMSZFaNwXv8OTk1kwOOgkpeB1Snp2sP6e17e4=
88114
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -188,8 +214,8 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj
188214
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
189215
github.com/go-git/go-git/v5 v5.16.3 h1:Z8BtvxZ09bYm/yYNgPKCzgWtaRqDTgIKRgIRHBfU6Z8=
190216
github.com/go-git/go-git/v5 v5.16.3/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8=
191-
github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI=
192-
github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo=
217+
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
218+
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
193219
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
194220
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
195221
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -561,8 +587,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
561587
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
562588
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
563589
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
564-
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
565-
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
590+
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
591+
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
566592
go.opentelemetry.io/contrib/bridges/otelzap v0.13.0 h1:aBKdhLVieqvwWe9A79UHI/0vgp2t/s2euY8X59pGRlw=
567593
go.opentelemetry.io/contrib/bridges/otelzap v0.13.0/go.mod h1:SYqtxLQE7iINgh6WFuVi2AI70148B8EI35DSk0Wr8m4=
568594
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 h1:YH4g8lQroajqUwWbq/tr2QX1JFmEXaDLgG+ew9bLMWo=
@@ -601,8 +627,8 @@ go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6
601627
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
602628
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
603629
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
604-
go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4=
605-
go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
630+
go.opentelemetry.io/proto/otlp v1.8.0 h1:fRAZQDcAFHySxpJ1TwlA1cJ4tvcrw7nXl9xWWC8N5CE=
631+
go.opentelemetry.io/proto/otlp v1.8.0/go.mod h1:tIeYOeNBU4cvmPqpaji1P+KbB4Oloai8wN4rWzRrFF0=
606632
go.starlark.net v0.0.0-20210406145628-7a1108eaa012/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0=
607633
go.starlark.net v0.0.0-20240314022150-ee8ed142361c h1:roAjH18hZcwI4hHStHbkXjF5b7UUyZ/0SG3hXNN1SjA=
608634
go.starlark.net v0.0.0-20240314022150-ee8ed142361c/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8=
@@ -719,18 +745,20 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA
719745
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
720746
google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
721747
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
722-
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY=
723-
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE=
724-
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE=
725-
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc=
748+
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4=
749+
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo=
750+
google.golang.org/genproto/googleapis/bytestream v0.0.0-20251111163417-95abcf5c77ba h1:4cGHLblHw/UoGn1a19HSoOP0+zuDK3pB+a2dVnBnS7s=
751+
google.golang.org/genproto/googleapis/bytestream v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:G3Q0qS3k/oFEmVMddPsSYcFnm2+Mq2XRmxujrtu5hr0=
752+
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk=
753+
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
726754
google.golang.org/grpc v1.0.5/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
727755
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
728756
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
729757
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
730758
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
731759
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
732-
google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
733-
google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
760+
google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
761+
google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
734762
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
735763
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
736764
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

‎internal/agent/http_cache/http_cache.go‎

Lines changed: 13 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"context"
66
"errors"
77
"fmt"
8-
"io"
98
"log/slog"
109
"net"
1110
"net/http"
@@ -19,6 +18,8 @@ import (
1918
"github.com/cirruslabs/cirrus-cli/internal/agent/http_cache/ghacache"
2019
"github.com/cirruslabs/cirrus-cli/internal/agent/http_cache/ghacachev2"
2120
"github.com/cirruslabs/cirrus-cli/pkg/api"
21+
"github.com/cirruslabs/omni-cache/pkg/storage"
22+
urlproxy "github.com/cirruslabs/omni-cache/pkg/url-proxy"
2223
sentryhttp "github.com/getsentry/sentry-go/http"
2324
"golang.org/x/sync/semaphore"
2425
"google.golang.org/grpc/codes"
@@ -34,6 +35,7 @@ const (
3435
type HTTPCache struct {
3536
httpClient *http.Client
3637
azureBlobOpts []azureblob.Option
38+
proxy *urlproxy.Proxy
3739
}
3840

3941
var sem = semaphore.NewWeighted(int64(runtime.NumCPU() * activeRequestsPerLogicalCPU))
@@ -56,11 +58,13 @@ func Start(
5658
transport = DefaultTransport()
5759
}
5860

61+
httpClient := &http.Client{
62+
Transport: transport,
63+
Timeout: 10 * time.Minute,
64+
}
5965
httpCache := &HTTPCache{
60-
httpClient: &http.Client{
61-
Transport: transport,
62-
Timeout: 10 * time.Minute,
63-
},
66+
httpClient: httpClient,
67+
proxy: urlproxy.NewProxy(urlproxy.WithHTTPClient(httpClient)),
6468
}
6569

6670
// Apply opts
@@ -200,47 +204,20 @@ func (httpCache *HTTPCache) downloadCache(w http.ResponseWriter, r *http.Request
200204
w.WriteHeader(http.StatusNotFound)
201205
} else {
202206
slog.Info("Redirecting cache download", "cache_key", cacheKey)
203-
httpCache.proxyDownloadFromURLs(w, r, response.Urls)
207+
httpCache.proxyDownloadFromURLs(w, r, cacheKey, response.Urls)
204208
}
205209
}
206210

207-
func (httpCache *HTTPCache) proxyDownloadFromURLs(w http.ResponseWriter, r *http.Request, urls []string) {
211+
func (httpCache *HTTPCache) proxyDownloadFromURLs(w http.ResponseWriter, r *http.Request, cacheKey string, urls []string) {
208212
for _, url := range urls {
209-
if httpCache.proxyDownloadFromURL(w, r, url) {
213+
urlInfo := storage.URLInfo{URL: url}
214+
if httpCache.proxy.ProxyDownloadFromURL(r.Context(), w, &urlInfo, cacheKey) {
210215
return
211216
}
212217
}
213218
w.WriteHeader(http.StatusNotFound)
214219
}
215220

216-
func (httpCache *HTTPCache) proxyDownloadFromURL(w http.ResponseWriter, r *http.Request, url string) bool {
217-
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, url, nil)
218-
if err != nil {
219-
slog.Error("Failed to create a new GET HTTP request", "url", url, "err", err)
220-
return false
221-
}
222-
resp, err := httpCache.httpClient.Do(req)
223-
if err != nil {
224-
slog.Error("Proxying cache failed", "url", url, "err", err)
225-
return false
226-
}
227-
defer resp.Body.Close()
228-
successfulStatus := 100 <= resp.StatusCode && resp.StatusCode < 300
229-
if !successfulStatus {
230-
slog.Warn("Proxying cache failed with non-success status", "url", url, "status_code", resp.StatusCode)
231-
return false
232-
}
233-
w.WriteHeader(resp.StatusCode)
234-
bytesRead, err := io.Copy(w, resp.Body)
235-
if err != nil {
236-
slog.Error("Proxying cache download failed", "url", url, "err", err)
237-
return false
238-
} else {
239-
slog.Info("Proxying cache succeeded", "url", url, "bytes", bytesRead)
240-
}
241-
return true
242-
}
243-
244221
func (httpCache *HTTPCache) uploadCacheEntry(w http.ResponseWriter, r *http.Request, cacheKey string) {
245222
key := api.CacheKey{
246223
TaskIdentification: client.CirrusTaskIdentification,

‎internal/agent/http_cache/http_cache_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ func TestHTTPCache(t *testing.T) {
3535
// Create the cache entry
3636
resp, err = http.Post(httpCacheObjectURL, "text/plain", strings.NewReader("Hello, World!"))
3737
require.NoError(t, err)
38-
require.Equal(t, http.StatusCreated, resp.StatusCode)
38+
require.Equal(t, http.StatusOK, resp.StatusCode)
3939

4040
// Ensure that the cache entry now exists
4141
resp, err = http.Head(httpCacheObjectURL)

‎internal/agent/http_cache/rpc_fallback.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ func (httpCache *HTTPCache) downloadCacheViaRPC(w http.ResponseWriter, r *http.R
4747

4848
if chunk.RedirectUrl != "" {
4949
slog.Info("Cache download via RPC requested redirect", "cache_key", cacheKey)
50-
httpCache.proxyDownloadFromURLs(w, r, []string{chunk.RedirectUrl})
50+
httpCache.proxyDownloadFromURLs(w, r, cacheKey, []string{chunk.RedirectUrl})
5151

5252
return
5353
}

0 commit comments

Comments
 (0)