Skip to content

Commit b4c88ae

Browse files
kordejongkordejong
authored
Merge pull request computationalgeography#930 from kordejong/gh588
Add support for passing creation options to to_gdal
2 parents a959ab7 + b9109de commit b4c88ae

File tree

8 files changed

+151
-26
lines changed

8 files changed

+151
-26
lines changed

‎source/data_model/gdal/include/lue/gdal/dataset.hpp‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#include "lue/gdal/define.hpp"
33
#include "lue/gdal/raster_band.hpp"
44
#include <gdal_priv.h>
5+
#include <map>
56
#include <memory>
67

78

@@ -41,19 +42,26 @@ namespace lue::gdal {
4142
std::string const& dataset_name,
4243
Shape const& shape,
4344
Count nr_bands,
44-
GDALDataType data_type) -> DatasetPtr;
45+
GDALDataType data_type,
46+
std::map<std::string, std::string> const& options = {}) -> DatasetPtr;
4547

4648
LUE_GDAL_EXPORT auto create_dataset(
4749
std::string const& driver_name,
4850
std::string const& dataset_name,
4951
Shape const& shape,
5052
Count nr_bands,
51-
GDALDataType data_type) -> DatasetPtr;
53+
GDALDataType data_type,
54+
std::map<std::string, std::string> const& options = {}) -> DatasetPtr;
5255

53-
LUE_GDAL_EXPORT auto create_dataset(std::string const& driver_name, std::string const& dataset_name)
54-
-> DatasetPtr;
56+
LUE_GDAL_EXPORT auto create_dataset(
57+
std::string const& driver_name,
58+
std::string const& dataset_name,
59+
std::map<std::string, std::string> const& options = {}) -> DatasetPtr;
5560

56-
LUE_GDAL_EXPORT auto create_copy(std::string const& name, GDALDataset& clone_dataset) -> DatasetPtr;
61+
LUE_GDAL_EXPORT auto create_copy(
62+
std::string const& name,
63+
GDALDataset& clone_dataset,
64+
std::map<std::string, std::string> const& options = {}) -> DatasetPtr;
5765

5866
LUE_GDAL_EXPORT auto delete_dataset(GDALDriver& driver, std::string const& dataset_name) -> void;
5967

‎source/data_model/gdal/source/dataset.cpp‎

Lines changed: 88 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22
#include "lue/gdal/driver.hpp"
33
#include <cassert>
44
#include <cmath>
5+
#include <cstring>
56
#include <format>
67
#include <stdexcept>
8+
#include <vector>
79

810

911
namespace lue::gdal {
@@ -21,6 +23,69 @@ namespace lue::gdal {
2123
return geo_transform[1] == std::abs(geo_transform[5]);
2224
}
2325

26+
27+
class Options
28+
{
29+
30+
public:
31+
32+
Options(std::map<std::string, std::string> const& options):
33+
34+
_dictionary{options},
35+
_buffers{}
36+
37+
{
38+
_buffers.reserve(_dictionary.size() + 1);
39+
40+
for (auto const& [name, value] : _dictionary)
41+
{
42+
std::string const string = std::format("{}={}", name, value);
43+
char* buffer = new char[string.size() + 1];
44+
std::strcpy(buffer, string.c_str());
45+
_buffers.emplace_back(buffer);
46+
47+
assert(std::strlen(_buffers.back()) == string.size());
48+
}
49+
50+
_buffers.push_back(nullptr);
51+
52+
assert(_buffers.size() == _dictionary.size() + 1);
53+
}
54+
55+
56+
Options(Options const& other) = delete;
57+
58+
Options(Options&& other) = delete;
59+
60+
61+
~Options()
62+
{
63+
for (char* buffer : _buffers)
64+
{
65+
// Yes, also deleting the final nullptr. It's fine.
66+
delete[] buffer;
67+
}
68+
}
69+
70+
71+
auto operator=(Options const& other) -> Options& = delete;
72+
73+
auto operator=(Options&& other) -> Options& = delete;
74+
75+
76+
auto buffers() -> char**
77+
{
78+
return _buffers.data();
79+
}
80+
81+
private:
82+
83+
std::map<std::string, std::string> _dictionary;
84+
85+
// Null-terminated vector of pointers to char buffers
86+
std::vector<char*> _buffers;
87+
};
88+
2489
} // Anonymous namespace
2590

2691

@@ -59,17 +124,21 @@ namespace lue::gdal {
59124
@param shape Shape of the raster bands in the dataset
60125
@param nr_bands Number of bands to add to the dataset
61126
@param data_type Type of the elements in the raster band(s)
127+
@param options Options to use when creating the dataset. See the GDAL driver's documentation for
128+
a list of these.
62129
@exception std::runtime_error In case the dataset cannot be created
63130
*/
64131
auto create_dataset(
65132
GDALDriver& driver,
66133
std::string const& dataset_name,
67134
Shape const& shape,
68135
Count nr_bands,
69-
GDALDataType data_type) -> DatasetPtr
136+
GDALDataType data_type,
137+
std::map<std::string, std::string> const& options) -> DatasetPtr
70138
{
71139
DatasetPtr dataset_ptr{
72-
driver.Create(dataset_name.c_str(), shape[1], shape[0], nr_bands, data_type, nullptr),
140+
driver.Create(
141+
dataset_name.c_str(), shape[1], shape[0], nr_bands, data_type, Options{options}.buffers()),
73142
gdal_close};
74143

75144
if (!dataset_ptr)
@@ -89,34 +158,45 @@ namespace lue::gdal {
89158
std::string const& dataset_name,
90159
Shape const& shape,
91160
Count nr_bands,
92-
GDALDataType data_type) -> DatasetPtr
161+
GDALDataType data_type,
162+
std::map<std::string, std::string> const& options) -> DatasetPtr
93163
{
94-
return create_dataset(*driver(driver_name), dataset_name, shape, nr_bands, data_type);
164+
return create_dataset(*driver(driver_name), dataset_name, shape, nr_bands, data_type, options);
95165
}
96166

97167

98168
/*!
99169
@overload
100170
*/
101-
auto create_dataset(std::string const& driver_name, std::string const& dataset_name) -> DatasetPtr
171+
auto create_dataset(
172+
std::string const& driver_name,
173+
std::string const& dataset_name,
174+
std::map<std::string, std::string> const& options) -> DatasetPtr
102175
{
103-
return create_dataset(driver_name, dataset_name, Shape{0, 0}, 0, GDT_Unknown);
176+
return create_dataset(driver_name, dataset_name, Shape{0, 0}, 0, GDT_Unknown, options);
104177
}
105178

106179

107180
/*!
108181
@brief Create a copy of a dataset
109182
@param name Name of the new dataset
110183
@param clone_dataset Dataset to copy
184+
@param options Options to use when creating the dataset. See the GDAL driver's documentation for
185+
a list of these.
111186
@exception std::runtime_error In case the dataset cannot be created
112187
*/
113-
auto create_copy(std::string const& name, GDALDataset& clone_dataset) -> DatasetPtr
188+
auto create_copy(
189+
std::string const& name,
190+
GDALDataset& clone_dataset,
191+
std::map<std::string, std::string> const& options) -> DatasetPtr
114192
{
115193
// TODO let GDAL pick the driver and/or use extension(?)
116194
DriverPtr driver{gdal::driver("GTiff")};
117195

118196
DatasetPtr dataset_ptr{
119-
driver->CreateCopy(name.c_str(), &clone_dataset, FALSE, nullptr, nullptr, nullptr), gdal_close};
197+
driver->CreateCopy(
198+
name.c_str(), &clone_dataset, FALSE, Options{options}.buffers(), nullptr, nullptr),
199+
gdal_close};
120200

121201
if (!dataset_ptr)
122202
{

‎source/framework/algorithm/include/lue/framework/algorithm/default_policies/normal.hpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ namespace lue {
6868
{
6969
using Policies = policy::normal::DefaultPolicies<Element>;
7070

71-
return lue::normal<Element>(Policies{}, array_shape, partition_shape, mean, stddev);
71+
return lue::normal(Policies{}, array_shape, partition_shape, mean, stddev);
7272
}
7373

7474

@@ -81,7 +81,7 @@ namespace lue {
8181
{
8282
using Policies = policy::normal::DefaultPolicies<Element>;
8383

84-
return lue::normal<Element>(Policies{}, array_shape, partition_shape, mean, stddev);
84+
return lue::normal(Policies{}, array_shape, partition_shape, mean, stddev);
8585
}
8686

8787

@@ -92,7 +92,7 @@ namespace lue {
9292
{
9393
using Policies = policy::normal::DefaultPolicies<Element>;
9494

95-
return lue::normal<Element>(Policies{}, array_shape, mean, stddev);
95+
return lue::normal(Policies{}, array_shape, mean, stddev);
9696
}
9797

9898

@@ -102,7 +102,7 @@ namespace lue {
102102
{
103103
using Policies = policy::normal::DefaultPolicies<Element>;
104104

105-
return lue::normal<Element>(Policies{}, array_shape, mean, stddev);
105+
return lue::normal(Policies{}, array_shape, mean, stddev);
106106
}
107107

108108
} // namespace default_policies

‎source/framework/core/include/lue/framework/core/hilbert_curve.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#pragma once
22
#include "lue/framework/core/shape.hpp"
3+
#include <cmath>
34

45

56
// The Hilbert curve algorithm is inspired by the Gilbert code:

‎source/framework/io/include/lue/framework/io/gdal.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ namespace lue {
1919
auto to_gdal(
2020
PartitionedArray<Element, 2> const& array,
2121
std::string const& name,
22-
std::string const& clone_name = "") -> hpx::future<void>;
22+
std::string const& clone_name = "",
23+
std::map<std::string, std::string> const& options = {}) -> hpx::future<void>;
2324

2425
} // namespace lue

‎source/framework/io/source/gdal.cpp.in‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -566,7 +566,8 @@ namespace lue {
566566
auto to_gdal(
567567
PartitionedArray<Element, 2> const& array,
568568
std::string const& name,
569-
std::string const& clone_name) -> hpx::future<void>
569+
std::string const& clone_name,
570+
std::map<std::string, std::string> const& options) -> hpx::future<void>
570571
{
571572
if constexpr (!gdal::supports<Element>())
572573
{
@@ -604,7 +605,8 @@ namespace lue {
604605
static_cast<gdal::Count>(raster_shape[0]),
605606
static_cast<gdal::Count>(raster_shape[1])},
606607
nr_bands,
607-
data_type)};
608+
data_type,
609+
options)};
608610
gdal::RasterBandPtr band_ptr{gdal::raster_band(*dataset_ptr)};
609611

610612
gdal::set_no_data_value(*band_ptr, no_data_value);
@@ -657,7 +659,7 @@ namespace lue {
657659

658660
hpx::future<void> result = hpx::async(
659661
[name, partitions = std::move(partitions)]() mutable
660-
{
662+
-> auto {
661663
using Action = WritePartitionAction<Policies, Partition>;
662664

663665
std::size_t nr_partitions_to_write{partitions.size()};
@@ -707,7 +709,7 @@ namespace lue {
707709
std::string const&, lue::Hyperslab<{{ rank }}> const&, lue::Shape<lue::Count, {{ rank }}> const&);
708710

709711
template LUE_FRAMEWORK_IO_EXPORT hpx::future<void> lue::to_gdal<{{ Element }}>(
710-
lue::PartitionedArray<{{ Element }}, {{ rank }}> const&, std::string const&, std::string const&);
712+
lue::PartitionedArray<{{ Element }}, {{ rank }}> const&, std::string const&, std::string const&, std::map<std::string, std::string> const&);
711713

712714
{% endfor %}
713715
{% endfor %}

‎source/framework/python/source/gdal/to_gdal.cpp‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#include "lue/framework.hpp"
33
#include "lue/gdal.hpp"
44
#include "lue/py/bind.hpp"
5+
#include <pybind11/stl.h>
56

67

78
using namespace pybind11::literals;
@@ -25,7 +26,26 @@ namespace lue::framework {
2526
{
2627
// If not one of the types not supported by older versions of GDAL OR using a GDAL
2728
// version that supports them ...
28-
module.def("to_gdal", to_gdal<Element>, "array"_a, "name"_a, "clone_name"_a = "");
29+
module.def(
30+
"to_gdal",
31+
to_gdal<Element>,
32+
R"(
33+
Write array to a dataset using GDAL
34+
35+
:param array: Array to write
36+
:param name: Name of dataset to write
37+
:param clone_name: Name of existing dataset to copy geo transform and
38+
spatial reference from
39+
:param dict options: Creation options (see GDAL driver documentation for
40+
a list)
41+
42+
The GTiff driver will be used to do the writing.
43+
)",
44+
"array"_a,
45+
"name"_a,
46+
pybind11::kw_only(),
47+
"clone_name"_a = "",
48+
"options"_a = std::map<std::string, std::string>{});
2949
}
3050
}
3151
};

‎source/framework/python/test/gdal_test.py‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ def tearDownModule():
1515
class GdalTest(lue_test.TestCase):
1616
@lue_test.framework_test_case
1717
def test_array_default_partition_shape(self):
18-
1918
array_shape = (600, 400)
2019
dtype = np.int32
2120
fill_value = 5
@@ -34,7 +33,6 @@ def test_array_default_partition_shape(self):
3433

3534
@lue_test.framework_test_case
3635
def test_array_custom_partition_shape(self):
37-
3836
array_shape = (600, 400)
3937
partition_shape = (10, 10)
4038
dtype = np.int32
@@ -56,7 +54,6 @@ def test_array_custom_partition_shape(self):
5654

5755
@lue_test.framework_test_case
5856
def test_hyperslab(self):
59-
6057
array_shape = (600, 400)
6158

6259
condition = lfr.create_array(array_shape, lfr.boolean_element_type, 1)
@@ -91,3 +88,19 @@ def test_hyperslab(self):
9188
# Verify the hyperslabs read contain values we expect
9289
self.assertTrue(lfr.all(row_idxs_read == row_idxs_hyperslab).future.get())
9390
self.assertTrue(lfr.all(col_idxs_read == col_idxs_hyperslab).future.get())
91+
92+
@lue_test.framework_test_case
93+
def test_creation_options(self):
94+
array_shape = (600, 400)
95+
dtype = np.int32
96+
fill_value = 5
97+
array_written = lfr.create_array(array_shape, dtype, fill_value)
98+
99+
name = "gdal_creation_options.tif"
100+
options = {
101+
"BIGTIFF": "YES",
102+
"COMPRESS": "ZSTD",
103+
}
104+
105+
written = lfr.to_gdal(array_written, name, options=options)
106+
written.get()

0 commit comments

Comments
 (0)