Skip to content

Commit ec986bd

Browse files
matamadiomatamadio
committed
Update utility: raster tile mosaicing
1 parent af2a3de commit ec986bd

File tree

2 files changed

+648
-0
lines changed

2 files changed

+648
-0
lines changed

‎tools/utility/merge_tiles/TileMosaicing.ipynb‎

Lines changed: 360 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,360 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"# General-Purpose Raster Tile Merger\n",
8+
"## TILES MOSAICING SCRIPT\n",
9+
"\n",
10+
"This Python script merges GeoTIFF tiles while intelligently handling nodata values.\n",
11+
"\n",
12+
"**Key Features:**\n",
13+
"- Preserves legitimate negative values (elevation, temperature, etc.)\n",
14+
"- Automatically detects nodata from tile metadata\n",
15+
"- Unifies multiple nodata values into a single value\n",
16+
"- Memory-efficient windowed processing for large files\n",
17+
"- Parallel processing for faster results\n",
18+
"\n",
19+
"**How to Use:**\n",
20+
"1. Specify the directory containing RP subfolders (not the RP subfolder itself)\n",
21+
"2. Choose merge mode:\n",
22+
" - **General**: Preserves negative values, detects nodata automatically\n",
23+
" - **Fathom**: Converts negatives to 0, sets nodata=0 (for flood depth data)\n",
24+
"3. Output files are created in the parent directory of each subfolder\n",
25+
"\n",
26+
"**Requirements:**\n",
27+
"- GDAL library must be installed in your Python environment\n",
28+
"- Works best on multi-core CPUs for parallel processing"
29+
]
30+
},
31+
{
32+
"cell_type": "code",
33+
"execution_count": null,
34+
"metadata": {},
35+
"outputs": [],
36+
"source": [
37+
"import os\n",
38+
"import sys\n",
39+
"import concurrent.futures\n",
40+
"import shutil\n",
41+
"import ipywidgets as widgets\n",
42+
"from IPython.display import display, clear_output\n",
43+
"import tkinter as tk\n",
44+
"from tkinter import filedialog\n",
45+
"\n",
46+
"# Add utility path to access merge_utils_general\n",
47+
"utility_path = os.path.abspath(os.path.join(os.getcwd(), '..', '..', 'utility'))\n",
48+
"if utility_path not in sys.path:\n",
49+
" sys.path.insert(0, utility_path)\n",
50+
"\n",
51+
"from merge_utils_general import merge_tifs_general, merge_tifs_fathom\n",
52+
"\n",
53+
"def select_folder():\n",
54+
" root = tk.Tk()\n",
55+
" root.attributes('-topmost', True)\n",
56+
" root.geometry(\"1x1+0+0\")\n",
57+
" root.withdraw()\n",
58+
" \n",
59+
" root.after(0, lambda: root.focus_force())\n",
60+
" folder_selected = filedialog.askdirectory(master=root)\n",
61+
" \n",
62+
" root.destroy()\n",
63+
" return folder_selected\n",
64+
"\n",
65+
"class GeneralTilesMosaicingTool:\n",
66+
" def __init__(self):\n",
67+
" self.directory_input = widgets.Text(\n",
68+
" value='',\n",
69+
" placeholder='Enter the directory path',\n",
70+
" description='Directory:',\n",
71+
" disabled=False,\n",
72+
" layout=widgets.Layout(width='500px')\n",
73+
" )\n",
74+
" \n",
75+
" self.search_button = widgets.Button(\n",
76+
" description='Search Folder',\n",
77+
" disabled=False,\n",
78+
" button_style='',\n",
79+
" tooltip='Click to search for a folder',\n",
80+
" layout=widgets.Layout(width='120px')\n",
81+
" )\n",
82+
" \n",
83+
" self.merge_mode = widgets.Dropdown(\n",
84+
" options=[\n",
85+
" ('General (preserve negatives, auto-detect nodata)', 'general'),\n",
86+
" ('Fathom (convert negatives to 0, nodata=0)', 'fathom')\n",
87+
" ],\n",
88+
" value='general',\n",
89+
" description='Merge Mode:',\n",
90+
" disabled=False,\n",
91+
" layout=widgets.Layout(width='500px')\n",
92+
" )\n",
93+
" \n",
94+
" self.target_nodata_input = widgets.Text(\n",
95+
" value='',\n",
96+
" placeholder='Leave empty for auto-detection',\n",
97+
" description='Target NoData:',\n",
98+
" disabled=False,\n",
99+
" layout=widgets.Layout(width='250px')\n",
100+
" )\n",
101+
" \n",
102+
" self.unify_nodata_checkbox = widgets.Checkbox(\n",
103+
" value=True,\n",
104+
" description='Unify multiple nodata values into target nodata',\n",
105+
" disabled=False,\n",
106+
" indent=False,\n",
107+
" layout=widgets.Layout(width='500px')\n",
108+
" )\n",
109+
" \n",
110+
" self.process_button = widgets.Button(\n",
111+
" description='Process Tiles',\n",
112+
" disabled=False,\n",
113+
" button_style='info',\n",
114+
" tooltip='Click to start processing',\n",
115+
" layout=widgets.Layout(width='120px')\n",
116+
" )\n",
117+
" \n",
118+
" self.delete_checkbox = widgets.Checkbox(\n",
119+
" value=False,\n",
120+
" description='Delete original subfolders after processing',\n",
121+
" disabled=False,\n",
122+
" indent=False,\n",
123+
" layout=widgets.Layout(width='500px')\n",
124+
" )\n",
125+
" \n",
126+
" self.output = widgets.Output()\n",
127+
" \n",
128+
" self.search_button.on_click(self.on_search_button_clicked)\n",
129+
" self.process_button.on_click(self.on_process_button_clicked)\n",
130+
" self.merge_mode.observe(self.on_merge_mode_change, names='value')\n",
131+
" \n",
132+
" # Layout\n",
133+
" display(widgets.VBox([\n",
134+
" widgets.HBox([self.directory_input, self.search_button]),\n",
135+
" self.merge_mode,\n",
136+
" widgets.HBox([self.target_nodata_input, self.unify_nodata_checkbox]),\n",
137+
" widgets.HBox([self.process_button, self.delete_checkbox]),\n",
138+
" self.output\n",
139+
" ]))\n",
140+
" \n",
141+
" def on_merge_mode_change(self, change):\n",
142+
" # Disable nodata options for Fathom mode\n",
143+
" if change['new'] == 'fathom':\n",
144+
" self.target_nodata_input.disabled = True\n",
145+
" self.unify_nodata_checkbox.disabled = True\n",
146+
" self.target_nodata_input.value = '0'\n",
147+
" else:\n",
148+
" self.target_nodata_input.disabled = False\n",
149+
" self.unify_nodata_checkbox.disabled = False\n",
150+
" \n",
151+
" def on_search_button_clicked(self, b):\n",
152+
" folder_path = select_folder()\n",
153+
" if folder_path:\n",
154+
" self.directory_input.value = folder_path\n",
155+
" \n",
156+
" def process_subdir(self, subdir):\n",
157+
" \"\"\"Process a single subdirectory\"\"\"\n",
158+
" try:\n",
159+
" if self.merge_mode.value == 'fathom':\n",
160+
" result = merge_tifs_fathom(subdir)\n",
161+
" else:\n",
162+
" # Parse target nodata\n",
163+
" target_nodata = None\n",
164+
" if self.target_nodata_input.value.strip():\n",
165+
" try:\n",
166+
" target_nodata = float(self.target_nodata_input.value.strip())\n",
167+
" except ValueError:\n",
168+
" print(f\"Warning: Invalid target nodata value '{self.target_nodata_input.value}'. Using auto-detection.\")\n",
169+
" \n",
170+
" result = merge_tifs_general(\n",
171+
" subdir,\n",
172+
" process_nodata=True,\n",
173+
" unify_nodata=self.unify_nodata_checkbox.value,\n",
174+
" target_nodata=target_nodata\n",
175+
" )\n",
176+
" return (True, subdir, result)\n",
177+
" except Exception as e:\n",
178+
" return (False, subdir, str(e))\n",
179+
" \n",
180+
" def on_process_button_clicked(self, b):\n",
181+
" self.output.clear_output()\n",
182+
" with self.output:\n",
183+
" directory = self.directory_input.value\n",
184+
" if not directory or not os.path.isdir(directory):\n",
185+
" print(f\"Error: {directory} is not a valid directory.\")\n",
186+
" return\n",
187+
" \n",
188+
" print(f\"Mode: {self.merge_mode.label}\")\n",
189+
" print(f\"Output files will be saved in: {directory}\")\n",
190+
" print()\n",
191+
" \n",
192+
" subdirs = [os.path.join(directory, subdir) for subdir in os.listdir(directory) \n",
193+
" if os.path.isdir(os.path.join(directory, subdir))]\n",
194+
" \n",
195+
" if not subdirs:\n",
196+
" print(\"No subdirectories found to process.\")\n",
197+
" return\n",
198+
" \n",
199+
" print(f\"Found {len(subdirs)} subdirectories to process.\")\n",
200+
" print()\n",
201+
" \n",
202+
" # Process subdirectories in parallel\n",
203+
" with concurrent.futures.ProcessPoolExecutor() as executor:\n",
204+
" results = executor.map(self.process_subdir, subdirs)\n",
205+
" \n",
206+
" # Collect results\n",
207+
" success_count = 0\n",
208+
" for success, subdir, result in results:\n",
209+
" if success:\n",
210+
" success_count += 1\n",
211+
" else:\n",
212+
" print(f\"ERROR processing {subdir}: {result}\")\n",
213+
" \n",
214+
" print()\n",
215+
" print(f\"Processing complete: {success_count}/{len(subdirs)} successful.\")\n",
216+
" \n",
217+
" if self.delete_checkbox.value:\n",
218+
" print()\n",
219+
" print(\"Deleting original subfolders...\")\n",
220+
" for subdir in subdirs:\n",
221+
" try:\n",
222+
" shutil.rmtree(subdir)\n",
223+
" print(f\"Deleted: {subdir}\")\n",
224+
" except Exception as e:\n",
225+
" print(f\"Error deleting {subdir}: {e}\")\n",
226+
" print(\"Original subfolders have been deleted.\")\n",
227+
" else:\n",
228+
" print(\"Original subfolders have been kept.\")\n",
229+
" \n",
230+
" print()\n",
231+
" print(\"Pre-processing completed.\")\n",
232+
"\n",
233+
"# Create and display the tool\n",
234+
"tool = GeneralTilesMosaicingTool()"
235+
]
236+
},
237+
{
238+
"cell_type": "markdown",
239+
"metadata": {},
240+
"source": [
241+
"# MANUAL RUN\n",
242+
"\n",
243+
"Use the cells below if you prefer to run the merger programmatically."
244+
]
245+
},
246+
{
247+
"cell_type": "code",
248+
"execution_count": null,
249+
"metadata": {},
250+
"outputs": [],
251+
"source": [
252+
"import os\n",
253+
"import sys\n",
254+
"import concurrent.futures\n",
255+
"import shutil\n",
256+
"\n",
257+
"# Add utility path\n",
258+
"utility_path = os.path.abspath(os.path.join(os.getcwd(), '..', '..', 'utility'))\n",
259+
"if utility_path not in sys.path:\n",
260+
" sys.path.insert(0, utility_path)\n",
261+
"\n",
262+
"from merge_utils_general import merge_tifs_general, merge_tifs_fathom\n",
263+
"\n",
264+
"# CONFIGURATION\n",
265+
"# Specify the directory containing the tile subfolders\n",
266+
"directory = 'C:/YourWorkDirectory/data/tiles/'\n",
267+
"\n",
268+
"# Choose merge function:\n",
269+
"# - merge_tifs_general: Preserves negative values, auto-detects nodata\n",
270+
"# - merge_tifs_fathom: Converts negatives to 0, sets nodata=0\n",
271+
"merge_function = merge_tifs_general\n",
272+
"\n",
273+
"# Optional: specify target nodata (None for auto-detection)\n",
274+
"target_nodata = None\n",
275+
"\n",
276+
"# Optional: unify multiple nodata values\n",
277+
"unify_nodata = True\n",
278+
"\n",
279+
"# Delete original subfolders after processing?\n",
280+
"delete_originals = False"
281+
]
282+
},
283+
{
284+
"cell_type": "code",
285+
"execution_count": null,
286+
"metadata": {},
287+
"outputs": [],
288+
"source": [
289+
"print(f\"Output files will be saved in: {directory}\")\n",
290+
"print()\n",
291+
"\n",
292+
"# Get subdirectories\n",
293+
"subdirs = [os.path.join(directory, subdir) for subdir in os.listdir(directory) \n",
294+
" if os.path.isdir(os.path.join(directory, subdir))]\n",
295+
"\n",
296+
"if not subdirs:\n",
297+
" print(\"No subdirectories found to process.\")\n",
298+
"else:\n",
299+
" print(f\"Found {len(subdirs)} subdirectories to process.\")\n",
300+
" print()\n",
301+
" \n",
302+
" # Process subdirectories\n",
303+
" def process_subdir(subdir):\n",
304+
" if merge_function == merge_tifs_fathom:\n",
305+
" return merge_tifs_fathom(subdir)\n",
306+
" else:\n",
307+
" return merge_tifs_general(subdir, process_nodata=True, \n",
308+
" unify_nodata=unify_nodata, \n",
309+
" target_nodata=target_nodata)\n",
310+
" \n",
311+
" # Process in parallel\n",
312+
" with concurrent.futures.ProcessPoolExecutor() as executor:\n",
313+
" results = list(executor.map(process_subdir, subdirs))\n",
314+
" \n",
315+
" print()\n",
316+
" print(\"Processing complete.\")\n",
317+
" \n",
318+
" # Delete originals if requested\n",
319+
" if delete_originals:\n",
320+
" user_input = input(\"Do you want to delete the original subfolders? (Y/N): \").strip().lower()\n",
321+
" if user_input == 'y':\n",
322+
" for subdir in subdirs:\n",
323+
" try:\n",
324+
" shutil.rmtree(subdir)\n",
325+
" print(f\"Deleted: {subdir}\")\n",
326+
" except Exception as e:\n",
327+
" print(f\"Error deleting {subdir}: {e}\")\n",
328+
" print(\"Original subfolders have been deleted.\")\n",
329+
" else:\n",
330+
" print(\"Original subfolders have been kept.\")\n",
331+
" else:\n",
332+
" print(\"Original subfolders have been kept.\")\n",
333+
" \n",
334+
" print()\n",
335+
" print(\"Pre-processing completed.\")"
336+
]
337+
}
338+
],
339+
"metadata": {
340+
"kernelspec": {
341+
"display_name": "Python 3",
342+
"language": "python",
343+
"name": "python3"
344+
},
345+
"language_info": {
346+
"codemirror_mode": {
347+
"name": "ipython",
348+
"version": 3
349+
},
350+
"file_extension": ".py",
351+
"mimetype": "text/x-python",
352+
"name": "python",
353+
"nbconvert_exporter": "python",
354+
"pygments_lexer": "ipython3",
355+
"version": "3.8.0"
356+
}
357+
},
358+
"nbformat": 4,
359+
"nbformat_minor": 4
360+
}

0 commit comments

Comments
 (0)