Current work on getting the wiki to be the source of truth.

Author: KockaAdmiralac

.gitignore

@@ -1,3 +1,4 @@
+*.mw-datamaps
 *.png
 *.tiled-session
 node_modules

extensions/includes/api.mjs

@@ -1,3 +1,4 @@
+import { InterwikiDataImpl, MetadataImpl } from './metadata.mjs';
 import { getStringProperty, getWikiUrl } from './util.mjs';
 
 const USER_AGENT = `tiled-datamaps/1.0 (https://github.com/utdrwiki/maps; admin@undertale.wiki) tiled/${tiled.version}`;
@@ -28,13 +29,18 @@ export function getRestUrl(language = 'en') {
  * @param {(value: any|PromiseLike<any>) => void} resolve Promise
  * resolution function
  * @param {(reason: any?) => void} reject Promise rejection function
+ * @param {boolean} isArrayBuffer Whether the request expects a binary response
  * @returns {() => void} Ready state change handler
  */
-const readyStateChange = (xhr, resolve, reject) => () => {
+const readyStateChange = (xhr, resolve, reject, isArrayBuffer = false) => () => {
     if (xhr.readyState === XMLHttpRequest.DONE) {
         if (xhr.status === 200) {
             try {
-                resolve(JSON.parse(xhr.responseText));
+                if (isArrayBuffer) {
+                    resolve(xhr.response);
+                } else {
+                    resolve(JSON.parse(xhr.responseText));
+                }
             } catch (error) {
                 reject(new Error(`Failed to parse response: ${xhr.responseText}`));
             }
@@ -160,3 +166,107 @@ export function edit(title, text, summary, accessToken, language = 'en') {
         return response.edit;
     });
 }
+
+/**
+ * Retrieves maps from the wiki under specific criteria.
+ * @param {object} options Options for retrieving maps
+ * @param {string} language Wiki language
+ * @returns {Promise<DataMap[]>} List of maps on the wiki
+ */
+function getMaps(options, language = 'en') {
+    return httpGet(getApiUrl(language), Object.assign({
+        action: 'query',
+        prop: 'revisions',
+        rvprop: 'ids|content',
+        rvslots: 'main',
+        format: 'json',
+        formatversion: '2',
+    }, options)).then(data => data.query.pages
+        .filter((/** @type {any} */ page) =>
+            page.revisions &&
+            page.revisions.length > 0 &&
+            page.revisions[0].slots &&
+            page.revisions[0].slots.main &&
+            page.revisions[0].slots.main.contentmodel === 'datamap'
+        )
+        .map((/** @type {any} */ page) => {
+            const {slots, revid} = page.revisions[0];
+            const /** @type {DataMap} */ datamap = JSON.parse(slots.main.content);
+            datamap.custom = datamap.custom || new MetadataImpl();
+            datamap.custom.interwiki = datamap.custom.interwiki || {};
+            datamap.custom.interwiki[language] = new InterwikiDataImpl({
+                mapName: page.title.split(':').slice(1).join(':'),
+            });
+            datamap.custom.interwiki[language].revision = revid;
+            return datamap;
+        })
+        .filter((/** @type {DataMap} */ datamap) => !datamap.$fragment)
+    );
+}
+
+/**
+ * Retrieves all maps from the wiki.
+ * @param {string} language Wiki language
+ * @returns {Promise<DataMap[]>} List of maps on the wiki
+ */
+export function getAllMaps(language = 'en') {
+    return getMaps({
+        generator: 'allpages',
+        gapnamespace: '2900',
+        gapfilterredir: 'nonredirects',
+        gaplimit: 'max',
+    }, language);
+}
+
+/**
+ * Retrieves a single map from the wiki.
+ * @param {string} name Map name
+ * @param {string} language Wiki language
+ * @returns {Promise<DataMap>} Specified map from the wiki
+ */
+export function getMap(name, language = 'en') {
+    return getMaps({
+        titles: `Map:${name}`
+    }, language).then(maps => maps[0]);
+}
+
+/**
+ * Returns the URLs of the given map files on the wiki.
+ * @param {string[]} filenames Map file names
+ * @param {string} language Wiki language
+ * @returns {Promise<string[]>} URLs of the given map files on the wiki
+ */
+export function getFileUrls(filenames, language = 'en') {
+    return httpGet(getApiUrl(language), {
+        action: 'query',
+        titles: filenames.map(name => `File:${name}`).join('|'),
+        prop: 'imageinfo',
+        iiprop: 'url',
+        format: 'json',
+        formatversion: '2'
+    }).then(data => filenames.map(filename => data.query.pages
+        .find((/** @type {any} */ page) => page.title === `File:${
+            data.query.normalized
+                ?.find((/** @type {any} */ n) => n.from === filename)
+                ?.to ||
+            filename
+        }`)
+        ?.imageinfo[0].url)
+        .filter(Boolean));
+}
+
+/**
+ * Downloads a file from a URL and returns it as an ArrayBuffer.
+ * @param {string} url URL to download the file from
+ * @returns {Promise<ArrayBuffer>} Downloaded file data
+ */
+export function downloadFile(url) {
+    return new Promise((resolve, reject) => {
+        const xhr = new XMLHttpRequest();
+        xhr.open('GET', url, true);
+        xhr.responseType = 'arraybuffer';
+        xhr.onreadystatechange = readyStateChange(xhr, resolve, reject, true);
+        xhr.setRequestHeader('User-Agent', USER_AGENT);
+        xhr.send();
+    });
+}

extensions/includes/format.mjs

@@ -7,7 +7,14 @@ import {
     getColorProperty,
     getListProperty,
     getNumberProperty,
-    getStringProperty
+    getStringProperty,
+    getTiledColor,
+    isBoxOverlay,
+    isImageBackground,
+    isPolylineOverlay,
+    setProperty,
+    validateTiledPoint,
+    validateTiledRectangle
 } from './util.mjs';
 
 /**
@@ -233,6 +240,108 @@ export function convertTiledToDataMaps(map, mapName, language = 'en') {
     return datamap;
 }
 
+/**
+ * Converts a DataMaps map to the Tiled map format.
+ * @param {DataMaps} datamaps DataMaps from all wikis to convert
+ * @returns {TileMap} Converted Tiled map object
+ */
+export function convertDataMapsToTiled(datamaps) {
+    const datamap = datamaps.en;
+    if (!datamap) {
+        throw new Error('English map data is required for conversion');
+    }
+    const metadata = new MetadataImpl(datamap.custom);
+    const primaryBg = datamap.backgrounds.find(isImageBackground);
+    if (!primaryBg) {
+        throw new Error('At least one background with an image is required to convert to Tiled format');
+    }
+    const crsBR = validateTiledPoint(datamap.crs.bottomRight);
+    const mapWidth = Math.ceil(crsBR[0] / metadata.tileWidth);
+    const mapHeight = Math.ceil(crsBR[1] / metadata.tileHeight);
+    const map = new TileMap();
+    map.tileWidth = metadata.tileWidth;
+    map.tileHeight = metadata.tileHeight;
+    map.width = mapWidth;
+    map.height = mapHeight;
+    if (datamap.disclaimer) {
+        map.setProperty('disclaimer', datamap.disclaimer);
+    }
+    if (datamap.settings && datamap.settings.leaflet) {
+        map.setProperty('popzoom', datamap.settings.leaflet.uriPopupZoom);
+    }
+    if (datamap.include) {
+        map.setProperty('include', datamap.include.join('\n'));
+    }
+    for (const bg of datamap.backgrounds.filter(isImageBackground)) {
+        const layer = new ImageLayer(bg.name || bg.image);
+        const fileName = metadata.getBackgroundFileName(bg.image);
+        const filePath = FileInfo.joinPaths(
+            tiled.project.folders[0],
+            'images',
+            fileName
+        );
+        layer.imageFileName = filePath;
+        layer.setProperty('image', bg.image);
+        map.addLayer(layer);
+    }
+    let annotationLayer = new ObjectGroup('annotations');
+    map.addLayer(annotationLayer);
+    for (const overlay of (primaryBg.overlays || [])) {
+        const obj = new MapObject(overlay.name);
+        if (isBoxOverlay(overlay)) {
+            obj.shape = MapObject.Rectangle;
+            const [[x, y], [x2, y2]] = validateTiledRectangle(overlay.at);
+            obj.pos = { x, y };
+            obj.width = x2 - x;
+            obj.height = y2 - y;
+            if (overlay.color) {
+                obj.setProperty('fill', getTiledColor(overlay.color));
+            }
+            if (overlay.borderColor) {
+                obj.setProperty('border', getTiledColor(overlay.borderColor));
+            }
+        } else if (isPolylineOverlay(overlay)) {
+            obj.polygon = overlay.path
+                .map(p => validateTiledPoint(p))
+                .map(p => ({ x: p[1], y: p[0] }));
+            obj.pos = { x: 0, y: 0 };
+            obj.shape = MapObject.Polyline;
+            if (overlay.color) {
+                obj.setProperty('color', getTiledColor(overlay.color));
+            }
+            if (overlay.thickness) {
+                obj.setProperty('thickness', overlay.thickness);
+            }
+        }
+        annotationLayer.addObject(obj);
+    }
+    // TODO: Support nested layers
+    for (const [layerName, markers] of Object.entries(datamap.markers).reverse()) {
+        const layer = new ObjectGroup(layerName);
+        for (const m of markers) {
+            const obj = new MapObject(m.name);
+            obj.pos = {
+                x: m.x,
+                y: m.y
+            };
+            obj.shape = MapObject.Point;
+            obj.setProperties({
+                page: m.article,
+                description: m.description,
+                image: m.image,
+                plain: m.isWikitext === undefined ? undefined : !m.isWikitext
+            });
+            layer.addObject(obj);
+        }
+        map.addLayer(layer);
+    }
+    for (const [language, interwiki] of Object.entries(metadata.interwiki)) {
+        setProperty(map, 'name', interwiki.mapName, language);
+        setProperty(map, 'revision', interwiki.revision, language);
+    }
+    return map;
+}
+
 /**
  * Converts a Tiled map to multiple DataMaps format for each language.
  * @param {TileMap} map Tiled map to convert to DataMaps
@@ -276,14 +385,30 @@ function generateWrite(multiple) {
     };
 }
 
+/**
+ * Converts a DataMaps map to the Tiled map format.
+ * @param {string} filePath Path to the file with the DataMaps map
+ * @returns {TileMap} Tiled map
+ */
+function read(filePath) {
+    const file = new TextFile(filePath, TextFile.ReadOnly);
+    const content = file.readAll();
+    file.close();
+    const datamap = JSON.parse(content);
+    const isMultiple = typeof datamap.en === 'object';
+    return convertDataMapsToTiled(isMultiple ? datamap : { en: datamap });
+}
+
 tiled.registerMapFormat('dataMaps', {
     extension: 'mw-datamaps',
     name: 'DataMaps (all wikis)',
+    read,
     write: generateWrite(true)
 });
 
 tiled.registerMapFormat('dataMap', {
     extension: 'mw-datamaps',
     name: 'DataMaps (single wiki)',
+    read,
     write: generateWrite(false)
 });