Skip to content

Commit b8aa9f2

Browse files
thisisaaronlandsfomuseumbot
andauthored
Add optional/experimental code to read PMTiles databases using System.FileDescriptor (#4)
* snapshot: block out code to use filedescriptor * add swifter-protomaps-server, untested * add UseFileDescriptor option --------- Co-authored-by: sfomuseumbot <sfomuseumbot@localhost>
1 parent a13f0f6 commit b8aa9f2

File tree

4 files changed

+237
-92
lines changed

4 files changed

+237
-92
lines changed

Package.resolved

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Package.swift

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ let package = Package(
1414
// .package(url: /* package url */, from: "1.0.0"),
1515
.package(url: "https://github.com/sfomuseum/swifter.git", branch:"main"),
1616
.package(url: "https://github.com/apple/swift-log.git", from: "1.6.2"),
17+
.package(url: "https://github.com/apple/swift-argument-parser", from: "1.5.0"),
1718
],
1819
targets: [
1920
// Targets are the basic building blocks of a package. A target can define a module or a test suite.
@@ -27,5 +28,15 @@ let package = Package(
2728
.testTarget(
2829
name: "SwifterProtomapsTests",
2930
dependencies: ["SwifterProtomaps"]),
31+
.executableTarget(
32+
name: "swifter-protomaps-server",
33+
dependencies: [
34+
"SwifterProtomaps",
35+
.product(name: "Swifter", package: "swifter"),
36+
.product(name: "Logging", package: "swift-log"),
37+
.product(name: "ArgumentParser", package: "swift-argument-parser")
38+
],
39+
path: "Scripts"
40+
)
3041
]
3142
)
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import ArgumentParser
2+
import Swifter
3+
import SwifterProtomaps
4+
import Logging
5+
import Foundation
6+
7+
@available(macOS 14.0, iOS 17.0, tvOS 17.0, *)
8+
@main
9+
struct SwifterProtomapsServer: ParsableCommand {
10+
11+
@Option(help: "The host name to listen for new connections")
12+
var host: String = "localhost"
13+
14+
@Option(help: "...")
15+
var root: String = ""
16+
17+
@Option(help: "The port to listen on for new connections")
18+
var port: UInt16 = 8080
19+
20+
@Option(help: "Enable verbose logging")
21+
var verbose: Bool = false
22+
23+
func run() throws {
24+
25+
let log_label = "org.sfomuseum.swift-protomaps"
26+
let logger = Logger(label: log_label)
27+
28+
let server = HttpServer()
29+
let root_url = URL(fileURLWithPath: root)
30+
31+
var opts = ServeProtomapsOptions(root: root_url)
32+
opts.AllowOrigins = "*"
33+
opts.AllowHeaders = "*"
34+
opts.Logger = logger
35+
36+
opts.StripPrefix = "/pmtiles"
37+
38+
server["/pmtiles/:path"] = ServeProtomapsTiles(opts)
39+
40+
server["/"] = { request in
41+
return HttpResponse.ok(.text("Hello world."))
42+
}
43+
44+
let semaphore = DispatchSemaphore(value: 0)
45+
46+
do {
47+
try server.start(port)
48+
logger.info("Server has started on \(port). Try to connect now...")
49+
semaphore.wait()
50+
} catch {
51+
logger.error("Server start error: \(error)")
52+
semaphore.signal()
53+
}
54+
55+
}
56+
}
Lines changed: 161 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import Swifter
22
import Foundation
33
import Logging
4+
import System
45

56
/// ServeProtomapsOptions defines runtime options for serving Protomaps tiles
67
public struct ServeProtomapsOptions {
@@ -14,135 +15,203 @@ public struct ServeProtomapsOptions {
1415
public var Logger: Logger?
1516
/// Optional string to strip from URL paths before processing
1617
public var StripPrefix: String
18+
/// Optional value to use System.FileDescriptor rather than Foundation.FileHandle to read data. Experimental.
19+
public var UseFileDescriptor: Bool
1720

1821
public init(root: URL) {
1922
Root = root
2023
AllowOrigins = ""
2124
AllowHeaders = ""
2225
StripPrefix = ""
26+
UseFileDescriptor = false
2327
}
2428
}
2529

2630
/// ServeProtomapsTiles will serve HTTP range requests for zero or more Protomaps tile databases in a directory.
2731
@available(iOS 13.4, *)
28-
@available(macOS 10.15.4, *)
32+
@available(macOS 11.0, *)
2933
public func ServeProtomapsTiles(_ opts: ServeProtomapsOptions) -> ((HttpRequest) -> HttpResponse) {
3034

31-
return { r in
32-
33-
var rsp_headers = [String: String]()
34-
35-
guard let req_path = r.params.first else {
36-
return .raw(404, "Not found", rsp_headers, {_ in })
37-
}
38-
39-
var rel_path = req_path.value
35+
return { r in
36+
37+
var rsp_headers = [String: String]()
38+
39+
guard let req_path = r.params.first else {
40+
return .raw(404, "Not found", rsp_headers, {_ in })
41+
}
42+
43+
var rel_path = req_path.value
44+
// opts.Logger?.info("Handle request \(rel_path)")
45+
46+
if opts.StripPrefix != "" {
47+
rel_path = rel_path.replacingOccurrences(of: opts.StripPrefix, with: "")
48+
}
49+
50+
let uri = opts.Root.appendingPathComponent(rel_path)
51+
let path = uri.absoluteString
52+
53+
// https://developer.apple.com/documentation/foundation/filehandle
54+
55+
var fh: FileHandle?
56+
var fd: FileDescriptor?
57+
58+
do {
4059

41-
if opts.StripPrefix != "" {
42-
rel_path = rel_path.replacingOccurrences(of: opts.StripPrefix, with: "")
60+
if opts.UseFileDescriptor {
61+
let fp = FilePath(uri.absoluteString.replacingOccurrences(of: "file://", with: ""))
62+
fd = try FileDescriptor.open(fp, .readOnly)
63+
} else {
64+
fh = try FileHandle(forReadingFrom: uri)
4365
}
4466

45-
let uri = opts.Root.appendingPathComponent(rel_path)
46-
let path = uri.absoluteString
47-
48-
// https://developer.apple.com/documentation/foundation/filehandle
49-
50-
var file: FileHandle
51-
67+
} catch {
68+
opts.Logger?.error("Failed to open path (\(path)) for reading \(error)")
69+
return .raw(404, "Not found", rsp_headers, {_ in })
70+
}
71+
72+
defer {
5273
do {
53-
file = try FileHandle(forReadingFrom: uri)
54-
} catch {
55-
opts.Logger?.error("Failed to open path (\(path)) for reading \(error)")
56-
return .raw(404, "Not found", rsp_headers, {_ in })
57-
}
58-
59-
defer {
60-
do {
61-
try file.close()
62-
} catch (let error) {
63-
opts.Logger?.warning("Failed to close \(path), \(error)")
74+
if opts.UseFileDescriptor {
75+
try fd?.close()
76+
} else {
77+
try fh?.close()
6478
}
79+
80+
} catch (let error) {
81+
opts.Logger?.warning("Failed to close \(path), \(error)")
6582
}
83+
}
84+
85+
guard var range_h = r.headers["range"] else {
86+
rsp_headers["Access-Control-Allow-Origin"] = opts.AllowOrigins
87+
rsp_headers["Access-Control-Allow-Headers"] = opts.AllowHeaders
88+
return .raw(200, "OK", rsp_headers, {_ in })
89+
}
90+
91+
let pat = "bytes=(\\d+)-(\\d+)"
92+
93+
guard let _ = range_h.range(of: pat, options: .regularExpression) else {
94+
rsp_headers["X-Error"] = "Invalid or unsupported range request"
95+
return .raw(400, "Bad Request", rsp_headers, {_ in })
96+
}
97+
98+
range_h = range_h.replacingOccurrences(of: "bytes=", with: "")
99+
let positions = range_h.split(separator: "-")
100+
101+
if positions.count != 2 {
102+
rsp_headers["X-Error"] = "Invalid count for range request"
103+
return .raw(400, "Bad Request", rsp_headers, {_ in })
104+
}
105+
106+
guard let start = UInt64(positions[0]) else {
107+
rsp_headers["X-Error"] = "Invalid starting range"
108+
return .raw(400, "Bad Request", rsp_headers, {_ in })
109+
}
110+
111+
guard let stop = Int(positions[1]) else {
112+
rsp_headers["X-Error"] = "Invalid stopping range"
113+
return .raw(400, "Bad Request", rsp_headers, {_ in })
114+
}
115+
116+
if start > stop {
117+
rsp_headers["X-Error"] = "Invalid range: Start value greater than stop value"
118+
return .raw(400, "Bad Request", rsp_headers, {_ in })
119+
}
120+
121+
let next = stop + 1
122+
123+
let body: Data!
124+
// file.seek(toFileOffset: start)
125+
126+
do {
66127

67-
guard var range_h = r.headers["range"] else {
68-
rsp_headers["Access-Control-Allow-Origin"] = opts.AllowOrigins
69-
rsp_headers["Access-Control-Allow-Headers"] = opts.AllowHeaders
70-
return .raw(200, "OK", rsp_headers, {_ in })
71-
}
72-
73-
let pat = "bytes=(\\d+)-(\\d+)"
74-
75-
guard let _ = range_h.range(of: pat, options: .regularExpression) else {
76-
rsp_headers["X-Error"] = "Invalid or unsupported range request"
77-
return .raw(400, "Bad Request", rsp_headers, {_ in })
78-
}
79-
80-
range_h = range_h.replacingOccurrences(of: "bytes=", with: "")
81-
let positions = range_h.split(separator: "-")
82-
83-
if positions.count != 2 {
84-
rsp_headers["X-Error"] = "Invalid count for range request"
85-
return .raw(400, "Bad Request", rsp_headers, {_ in })
86-
}
87-
88-
guard let start = UInt64(positions[0]) else {
89-
rsp_headers["X-Error"] = "Invalid starting range"
90-
return .raw(400, "Bad Request", rsp_headers, {_ in })
91-
}
92-
93-
guard let stop = Int(positions[1]) else {
94-
rsp_headers["X-Error"] = "Invalid stopping range"
95-
return .raw(400, "Bad Request", rsp_headers, {_ in })
128+
if opts.UseFileDescriptor {
129+
try fd?.seek(offset: Int64(start), from: FileDescriptor.SeekOrigin.start)
130+
} else {
131+
fh?.seek(toFileOffset: start)
96132
}
97133

98-
if start > stop {
99-
rsp_headers["X-Error"] = "Invalid range: Start value greater than stop value"
100-
return .raw(400, "Bad Request", rsp_headers, {_ in })
134+
} catch {
135+
opts.Logger?.error("Failed to seek to \(start) for \(path), \(error)")
136+
rsp_headers["X-Error"] = "Failed to read from Protomaps tile"
137+
return .raw(500, "Internal Server Error", rsp_headers, {_ in })
138+
}
139+
140+
if opts.UseFileDescriptor {
141+
142+
guard let data = readData(from: fd!.rawValue, length: next) else {
143+
opts.Logger?.error("Failed to read to \(next) for \(path)")
144+
rsp_headers["X-Error"] = "Failed to read from Protomaps tile"
145+
return .raw(500, "Internal Server Error", rsp_headers, {_ in })
101146
}
102147

103-
let next = stop + 1
148+
body = data
104149

105-
let body: Data!
106-
107-
file.seek(toFileOffset: start)
150+
} else {
108151

109152
do {
110-
body = try file.read(upToCount: next)
153+
body = try fh?.read(upToCount: next)
111154
} catch (let error){
112155
opts.Logger?.error("Failed to read to \(next) for \(path), \(error)")
113156
rsp_headers["X-Error"] = "Failed to read from Protomaps tile"
114157
return .raw(500, "Internal Server Error", rsp_headers, {_ in })
115158
}
116-
117-
// https://httpwg.org/specs/rfc7233.html#header.accept-ranges
118-
119-
var filesize = "*"
120159

160+
}
161+
162+
// https://httpwg.org/specs/rfc7233.html#header.accept-ranges
163+
164+
var filesize = "*"
165+
166+
if opts.UseFileDescriptor {
167+
() // pass for now
168+
} else {
121169
do {
122-
let size = try file.seekToEnd()
170+
let size = try fh!.seekToEnd()
123171
filesize = String(size)
124172
} catch (let error){
125173
opts.Logger?.warning("Failed to determine filesize for \(path), \(error)")
126174
}
127-
128-
let length = UInt64(next) - start
129-
130-
let content_length = String(length)
131-
let content_range = "bytes \(start)-\(next)/\(filesize)"
132-
133-
rsp_headers["Access-Control-Allow-Origin"] = opts.AllowOrigins
134-
rsp_headers["Access-Control-Allow-Headers"] = opts.AllowHeaders
135-
rsp_headers["Content-Length"] = content_length
136-
rsp_headers["Content-Range"] = content_range
137-
rsp_headers["Accept-Ranges"] = "bytes"
138-
139-
return .raw(206, "Partial Content", rsp_headers, { writer in
140-
141-
do {
142-
try writer.write(body)
143-
} catch (let error) {
144-
opts.Logger?.error("Failed to write body, \(error)")
145-
}
146-
})
147175
}
176+
177+
let length = UInt64(next) - start
178+
179+
let content_length = String(length)
180+
let content_range = "bytes \(start)-\(next)/\(filesize)"
181+
182+
rsp_headers["Access-Control-Allow-Origin"] = opts.AllowOrigins
183+
rsp_headers["Access-Control-Allow-Headers"] = opts.AllowHeaders
184+
rsp_headers["Content-Length"] = content_length
185+
rsp_headers["Content-Range"] = content_range
186+
rsp_headers["Accept-Ranges"] = "bytes"
187+
188+
return .raw(206, "Partial Content", rsp_headers, { writer in
189+
190+
do {
191+
try writer.write(body)
192+
} catch (let error) {
193+
opts.Logger?.error("Failed to write body, \(error)")
194+
}
195+
})
196+
}
197+
}
198+
199+
internal func readData(from fileDescriptor: Int32, length: Int) -> Data? {
200+
// Create a Data buffer of the desired length
201+
var data = Data(count: length)
202+
203+
// Read the data into the Data buffer
204+
let bytesRead = data.withUnsafeMutableBytes { buffer -> Int in
205+
guard let baseAddress = buffer.baseAddress else { return -1 }
206+
return read(fileDescriptor, baseAddress, length)
207+
}
208+
209+
// Handle errors or end-of-file
210+
guard bytesRead > 0 else {
211+
return nil // Return nil if no bytes were read
148212
}
213+
214+
// Resize the Data object to the actual number of bytes read
215+
data.removeSubrange(bytesRead..<data.count)
216+
return data
217+
}

0 commit comments

Comments
 (0)