Skip to content

Commit 7c45537

Browse files
lonvialonvia
authored
Merge pull request #311 from lonvia/reorganise-threadpools
Introduce explicit thread pool management
2 parents b73e223 + 759e6f3 commit 7c45537

23 files changed

+482
-223
lines changed

‎docs/reference/IO.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@
55
::: osmium.io.Header
66
::: osmium.io.Reader
77
::: osmium.io.Writer
8-
8+
::: osmium.io.ThreadPool

‎lib/base_handler.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
*
33
* This file is part of pyosmium. (https://osmcode.org/pyosmium/)
44
*
5-
* Copyright (C) 2024 Sarah Hoffmann <[email protected]> and others.
5+
* Copyright (C) 2025 Sarah Hoffmann <[email protected]> and others.
66
* For a full list of authors see the git log.
77
*/
88
#ifndef PYOSMIUM_BASE_HANDLER_HPP
@@ -41,7 +41,6 @@ class BaseHandler
4141
osmium::osm_entity_bits::type m_enabled_for = osmium::osm_entity_bits::all;
4242
};
4343

44-
void apply(osmium::io::Reader &reader, BaseHandler &handler);
4544
void apply_item(osmium::OSMEntity &item, BaseHandler &handler);
4645

4746
} // namespace

‎lib/file_iterator.cc‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
*
33
* This file is part of pyosmium. (https://osmcode.org/pyosmium/)
44
*
5-
* Copyright (C) 2024 Sarah Hoffmann <[email protected]> and others.
5+
* Copyright (C) 2025 Sarah Hoffmann <[email protected]> and others.
66
* For a full list of authors see the git log.
77
*/
88
#include <pybind11/pybind11.h>
@@ -15,6 +15,7 @@
1515
#include "osm_base_objects.h"
1616
#include "handler_chain.h"
1717
#include "python_handler.h"
18+
#include "io.h"
1819

1920
namespace py = pybind11;
2021

@@ -23,8 +24,8 @@ namespace {
2324
class OsmFileIterator
2425
{
2526
public:
26-
OsmFileIterator(osmium::io::Reader *reader, py::args args)
27-
: m_reader(reader), m_handler(args)
27+
OsmFileIterator(pyosmium::PyReader &reader, py::args args)
28+
: m_reader(reader.get()), m_handler(args)
2829
{
2930
m_buffer = m_reader->read();
3031

@@ -140,12 +141,12 @@ namespace pyosmium {
140141
void init_osm_file_iterator(py::module &m)
141142
{
142143
py::class_<OsmFileIterator>(m, "OsmFileIterator")
143-
.def(py::init<osmium::io::Reader *, py::args>(),
144-
py::keep_alive<0, 1>())
144+
.def(py::init<pyosmium::PyReader &, py::args>(),
145+
py::keep_alive<1, 2>())
145146
.def("set_filtered_handler", &OsmFileIterator::set_filtered_handler,
146-
py::keep_alive<0, 1>())
147+
py::keep_alive<1, 2>())
147148
.def("set_filtered_handler", &OsmFileIterator::set_filtered_python_handler,
148-
py::keep_alive<0, 1>())
149+
py::keep_alive<1, 2>())
149150
.def("__iter__", [](py::object const &self) { return self; })
150151
.def("__next__", &OsmFileIterator::next)
151152
;

‎lib/id_tracker.cc‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
*
33
* This file is part of pyosmium. (https://osmcode.org/pyosmium/)
44
*
5-
* Copyright (C) 2024 Sarah Hoffmann <[email protected]> and others.
5+
* Copyright (C) 2025 Sarah Hoffmann <[email protected]> and others.
66
* For a full list of authors see the git log.
77
*/
88
#include <pybind11/pybind11.h>
@@ -12,6 +12,7 @@
1212
#include <osmium/io/any_input.hpp>
1313
#include <osmium/index/nwr_array.hpp>
1414
#include <osmium/index/id_set.hpp>
15+
#include <osmium/thread/pool.hpp>
1516

1617
#include "base_filter.h"
1718
#include "osmium_module.h"
@@ -100,10 +101,11 @@ class IdTracker
100101

101102
void complete_backward_references(osmium::io::File file, int relation_depth)
102103
{
104+
osmium::thread::Pool thread_pool{};
103105
// first pass: relations
104106
while (relation_depth > 0 && !m_ids.relations().empty()) {
105107
bool need_recurse = false;
106-
osmium::io::Reader rd{file, osmium::osm_entity_bits::relation};
108+
osmium::io::Reader rd{file, osmium::osm_entity_bits::relation, thread_pool};
107109
while (auto const buffer = rd.read()) {
108110
for (auto const &rel: buffer.select<osmium::Relation>()) {
109111
if (m_ids.relations().get(rel.id())) {
@@ -125,7 +127,7 @@ class IdTracker
125127

126128
// second pass: ways
127129
if (!m_ids.ways().empty()) {
128-
osmium::io::Reader rd{file, osmium::osm_entity_bits::way};
130+
osmium::io::Reader rd{file, osmium::osm_entity_bits::way, thread_pool};
129131
while (auto const buffer = rd.read()) {
130132
for (auto const &way: buffer.select<osmium::Way>()) {
131133
if (m_ids.ways().get(way.id())) {
@@ -141,13 +143,14 @@ class IdTracker
141143

142144
void complete_forward_references(osmium::io::File file, int relation_depth)
143145
{
146+
osmium::thread::Pool thread_pool{};
144147
// standard pass: find directly referenced ways and relations
145148
{
146149
auto entities = osmium::osm_entity_bits::way;
147150
if (relation_depth >= 0) {
148151
entities |= osmium::osm_entity_bits::relation;
149152
}
150-
osmium::io::Reader rd{file, entities};
153+
osmium::io::Reader rd{file, entities, thread_pool};
151154
while (auto const buffer = rd.read()) {
152155
for (auto const &object: buffer.select<osmium::OSMObject>()) {
153156
if (object.type() == osmium::item_type::way) {
@@ -176,7 +179,7 @@ class IdTracker
176179
// recursive passes: find additional referenced relations
177180
while (relation_depth > 0 && !m_ids.relations().empty()) {
178181
bool need_recurse = false;
179-
osmium::io::Reader rd{file, osmium::osm_entity_bits::relation};
182+
osmium::io::Reader rd{file, osmium::osm_entity_bits::relation, thread_pool};
180183
while (auto const buffer = rd.read()) {
181184
for (auto const &rel: buffer.select<osmium::Relation>()) {
182185
if (!m_ids.relations().get(rel.id())) {

‎lib/io.cc‎

Lines changed: 60 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,12 @@
1010

1111
#include <osmium/io/any_input.hpp>
1212
#include <osmium/io/any_output.hpp>
13+
#include <osmium/thread/pool.hpp>
1314

1415
#include <filesystem>
1516

17+
#include "io.h"
18+
1619
namespace py = pybind11;
1720

1821
namespace {
@@ -78,34 +81,65 @@ PYBIND11_MODULE(io, m)
7881
py::return_value_policy::reference_internal)
7982
;
8083

81-
py::class_<osmium::io::Reader>(m, "Reader")
82-
.def(py::init<std::string>())
83-
.def(py::init<std::string, osmium::osm_entity_bits::type>())
84-
.def(py::init<>([] (std::filesystem::path const &file) {
85-
return new osmium::io::Reader(file.string());
86-
}))
87-
.def(py::init<>([] (std::filesystem::path const &file, osmium::osm_entity_bits::type etype) {
88-
return new osmium::io::Reader(file.string(), etype);
89-
}))
90-
.def(py::init<osmium::io::File>(),
91-
py::keep_alive<1, 2>())
92-
.def(py::init<osmium::io::File, osmium::osm_entity_bits::type>(),
93-
py::keep_alive<1, 2>())
94-
.def("eof", &osmium::io::Reader::eof)
95-
.def("close", &osmium::io::Reader::close)
96-
.def("header", &osmium::io::Reader::header)
84+
py::class_<pyosmium::PyReader>(m, "Reader")
85+
.def(py::init<osmium::io::File, osmium::osm_entity_bits::type const *,
86+
osmium::thread::Pool *>(),
87+
py::keep_alive<1, 2>(), py::keep_alive<1, 4>(),
88+
py::arg("file"), py::arg("types") = nullptr, py::arg("thread_pool") = nullptr
89+
)
90+
.def(py::init<>([] (std::string file,
91+
osmium::osm_entity_bits::type const *types,
92+
osmium::thread::Pool *pool) {
93+
return new pyosmium::PyReader(osmium::io::File(std::move(file)),
94+
types, pool); }),
95+
py::keep_alive<1, 2>(), py::keep_alive<1, 4>(),
96+
py::arg("file"), py::arg("types") = nullptr, py::arg("thread_pool") = nullptr
97+
)
98+
.def(py::init<>([] (std::filesystem::path const &file,
99+
osmium::osm_entity_bits::type const *types,
100+
osmium::thread::Pool *pool) {
101+
return new pyosmium::PyReader(osmium::io::File(file.string()),
102+
types, pool); }),
103+
py::keep_alive<1, 2>(), py::keep_alive<1, 4>(),
104+
py::arg("file"), py::arg("types") = nullptr, py::arg("thread_pool") = nullptr
105+
)
106+
.def("eof", [](pyosmium::PyReader const &self) { return self.get()->eof(); })
107+
.def("close", [](pyosmium::PyReader &self) { self.get()->close(); })
108+
.def("header", [](pyosmium::PyReader &self) { return self.get()->header(); })
97109
.def("__enter__", [](py::object const &self) { return self; })
98-
.def("__exit__", [](osmium::io::Reader &self, py::args args) { self.close(); })
110+
.def("__exit__", [](pyosmium::PyReader &self, py::args args) { self.get()->close(); })
99111
;
100112

101-
py::class_<osmium::io::Writer>(m, "Writer")
102-
.def(py::init<std::string>())
103-
.def(py::init<>([] (std::filesystem::path const &file) {
104-
return new osmium::io::Writer(file.string());
105-
}))
106-
.def(py::init<osmium::io::File>())
107-
.def(py::init<std::string, osmium::io::Header>())
108-
.def(py::init<osmium::io::File, osmium::io::Header>())
109-
.def("close", &osmium::io::Writer::close)
113+
py::class_<pyosmium::PyWriter>(m, "Writer")
114+
.def(py::init<osmium::io::File, osmium::io::Header const *, bool, osmium::thread::Pool *>(),
115+
py::keep_alive<1, 5>(),
116+
py::arg("file"), py::arg("header") = nullptr,
117+
py::arg("overwrite") = false, py::arg("thread_pool") = nullptr
118+
)
119+
.def(py::init<>([] (std::filesystem::path const &file, osmium::io::Header const *header,
120+
bool overwrite, osmium::thread::Pool *pool) {
121+
return new pyosmium::PyWriter(osmium::io::File(file.string()),
122+
header, overwrite, pool); }),
123+
py::keep_alive<1, 5>(),
124+
py::arg("file"), py::arg("header") = nullptr,
125+
py::arg("overwrite") = false, py::arg("thread_pool") = nullptr
126+
)
127+
.def(py::init<>([] (std::string filename, osmium::io::Header const *header,
128+
bool overwrite, osmium::thread::Pool *pool) {
129+
return new pyosmium::PyWriter(osmium::io::File(std::move(filename)),
130+
header, overwrite, pool); }),
131+
py::keep_alive<1, 5>(),
132+
py::arg("file"), py::arg("header") = nullptr,
133+
py::arg("overwrite") = false, py::arg("thread_pool") = nullptr
134+
)
135+
.def("close", [](pyosmium::PyWriter &self) { self.get()->close(); })
110136
;
137+
138+
py::class_<osmium::thread::Pool>(m, "ThreadPool")
139+
.def(py::init<int, std::size_t>(),
140+
py::arg("num_threads")=0, py::arg("max_queue_size")=0U)
141+
.def_property_readonly("num_threads", &osmium::thread::Pool::num_threads)
142+
.def("queue_size", &osmium::thread::Pool::queue_size)
143+
.def("queue_empty", &osmium::thread::Pool::queue_empty)
144+
;
111145
}

‎lib/io.h‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/* SPDX-License-Identifier: BSD-2-Clause
2+
*
3+
* This file is part of pyosmium. (https://osmcode.org/pyosmium/)
4+
*
5+
* Copyright (C) 2025 Sarah Hoffmann <[email protected]> and others.
6+
* For a full list of authors see the git log.
7+
*/
8+
#ifndef PYOSMIUM_IO_H
9+
#define PYOSMIUM_IO_H
10+
11+
#include <osmium/thread/pool.hpp>
12+
#include <osmium/io/any_input.hpp>
13+
#include <osmium/io/any_output.hpp>
14+
15+
namespace pyosmium {
16+
17+
class PyReader
18+
{
19+
public:
20+
explicit PyReader(osmium::io::File fname)
21+
: thread_pool(std::make_unique<osmium::thread::Pool>()),
22+
reader(std::move(fname))
23+
{}
24+
25+
PyReader(osmium::io::File fname, osmium::osm_entity_bits::type const *etype,
26+
osmium::thread::Pool *pool)
27+
: thread_pool(pool ? std::unique_ptr<osmium::thread::Pool>()
28+
: std::make_unique<osmium::thread::Pool>()),
29+
reader(std::move(fname),
30+
etype ? *etype : osmium::osm_entity_bits::all,
31+
*(pool ? pool : thread_pool.get()))
32+
{}
33+
34+
osmium::io::Reader const *get() const { return &reader; }
35+
osmium::io::Reader *get() { return &reader; }
36+
37+
private:
38+
std::unique_ptr<osmium::thread::Pool> thread_pool;
39+
osmium::io::Reader reader;
40+
};
41+
42+
43+
class PyWriter
44+
{
45+
public:
46+
PyWriter(osmium::io::File file, osmium::io::Header const *header,
47+
bool overwrite, osmium::thread::Pool *pool)
48+
: thread_pool(pool ? std::unique_ptr<osmium::thread::Pool>()
49+
: std::make_unique<osmium::thread::Pool>()),
50+
writer(std::move(file),
51+
header ? *header : osmium::io::Header(),
52+
overwrite ? osmium::io::overwrite::allow : osmium::io::overwrite::no,
53+
*(pool ? pool : thread_pool.get()))
54+
{}
55+
56+
osmium::io::Writer const *get() const { return &writer; }
57+
osmium::io::Writer *get() { return &writer; }
58+
59+
private:
60+
std::unique_ptr<osmium::thread::Pool> thread_pool;
61+
osmium::io::Writer writer;
62+
};
63+
64+
}
65+
66+
#endif // PYOSMIUM_IO_H

‎lib/merge_input_reader.cc‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,26 @@
22
*
33
* This file is part of pyosmium. (https://osmcode.org/pyosmium/)
44
*
5-
* Copyright (C) 2024 Sarah Hoffmann <[email protected]> and others.
5+
* Copyright (C) 2025 Sarah Hoffmann <[email protected]> and others.
66
* For a full list of authors see the git log.
77
*/
88
#include <pybind11/pybind11.h>
99

1010
#include <vector>
1111

12-
#include <boost/function_output_iterator.hpp>
12+
#include <boost/iterator/function_output_iterator.hpp>
1313

1414
#include <osmium/osm/object_comparisons.hpp>
1515
#include <osmium/io/any_input.hpp>
1616
#include <osmium/io/any_output.hpp>
1717
#include <osmium/io/output_iterator.hpp>
1818
#include <osmium/object_pointer_collection.hpp>
1919
#include <osmium/visitor.hpp>
20+
#include <osmium/thread/pool.hpp>
2021

2122
#include "osmium_module.h"
2223
#include "handler_chain.h"
24+
#include "io.h"
2325

2426
namespace py = pybind11;
2527

@@ -82,16 +84,16 @@ class MergeInputReader
8284
changes.clear();
8385
}
8486

85-
void apply_to_reader(osmium::io::Reader &reader, osmium::io::Writer &writer,
87+
void apply_to_reader(pyosmium::PyReader &reader, pyosmium::PyWriter &writer,
8688
bool with_history)
8789
{
88-
auto input = osmium::io::make_input_iterator_range<osmium::OSMObject>(reader);
90+
auto input = osmium::io::make_input_iterator_range<osmium::OSMObject>(*reader.get());
8991
if (with_history) {
9092
// For history files this is a straightforward sort of the change
9193
// files followed by a merge with the input file.
9294
objects.sort(osmium::object_order_type_id_version());
9395

94-
auto out = osmium::io::make_output_iterator(writer);
96+
auto out = osmium::io::make_output_iterator(*writer.get());
9597
std::set_union(objects.begin(),
9698
objects.end(),
9799
input.begin(),
@@ -111,7 +113,7 @@ class MergeInputReader
111113
objects.sort(osmium::object_order_type_id_reverse_version());
112114

113115
auto output_it = boost::make_function_output_iterator(
114-
copy_first_with_id(writer)
116+
copy_first_with_id(*writer.get())
115117
);
116118

117119
std::set_union(objects.begin(),
@@ -144,8 +146,9 @@ class MergeInputReader
144146
size_t internal_add(osmium::io::File change_file)
145147
{
146148
size_t sz = 0;
149+
osmium::thread::Pool thread_pool{};
147150

148-
osmium::io::Reader reader(change_file, osmium::osm_entity_bits::nwr);
151+
osmium::io::Reader reader(change_file, osmium::osm_entity_bits::nwr, thread_pool);
149152
while (osmium::memory::Buffer buffer = reader.read()) {
150153
osmium::apply(buffer, objects);
151154
sz += buffer.committed();

0 commit comments

Comments
 (0)