Current work on getting the wiki to be the source of truth.

Author: KockaAdmiralac

extensions/datamaps.mjs

@@ -1,12 +1,15 @@
 import dataMapsFormat from './includes/format.mjs';
 import selectedObjectsChanged from './includes/popup.mjs';
 import publishToWiki from './includes/publish.mjs';
+import pullFromWiki from './includes/pull.mjs';
 
 tiled.registerMapFormat('dataMaps', dataMapsFormat);
 const publishAction = tiled.registerAction('PublishToWiki', publishToWiki);
 publishAction.text = 'Publish to wiki';
 publishAction.icon = 'wiki.svg';
 publishAction.shortcut = 'Ctrl+Shift+U';
+const pullAction = tiled.registerAction('PullFromWiki', pullFromWiki);
+pullAction.text = 'Pull from wiki';
 const enablePopup = tiled.registerAction('WikiMarkerPopup', () => {});
 enablePopup.checkable = true;
 enablePopup.checked = true;
@@ -16,6 +19,9 @@ enablePopup.shortcut = 'Ctrl+Shift+M';
 tiled.extendMenu('File', [
     {
         action: 'PublishToWiki'
+    },
+    {
+        action: 'PullFromWiki'
     }
 ]);
 tiled.extendMenu('Edit', [

extensions/includes/api.mjs

@@ -1,3 +1,4 @@
+import { InterwikiDataImpl, MetadataImpl } from './metadata.mjs';
 import { getStringProperty, getWikiUrl } from './util.mjs';
 
 const USER_AGENT = `tiled-datamaps/1.0 (https://github.com/utdrwiki/maps; admin@undertale.wiki) tiled/${tiled.version}`;
@@ -28,13 +29,18 @@ export function getRestUrl(language = 'en') {
  * @param {(value: any|PromiseLike<any>) => void} resolve Promise
  * resolution function
  * @param {(reason: any?) => void} reject Promise rejection function
+ * @param {boolean} isArrayBuffer Whether the request expects a binary response
  * @returns {() => void} Ready state change handler
  */
-const readyStateChange = (xhr, resolve, reject) => () => {
+const readyStateChange = (xhr, resolve, reject, isArrayBuffer = false) => () => {
     if (xhr.readyState === XMLHttpRequest.DONE) {
         if (xhr.status === 200) {
             try {
-                resolve(JSON.parse(xhr.responseText));
+                if (isArrayBuffer) {
+                    resolve(xhr.response);
+                } else {
+                    resolve(JSON.parse(xhr.responseText));
+                }
             } catch (error) {
                 reject(new Error(`Failed to parse response: ${xhr.responseText}`));
             }
@@ -159,3 +165,77 @@ export function edit(title, text, summary, accessToken, language = 'en') {
         })
     );
 }
+
+/**
+ * Retrieves all maps from the wiki.
+ * @param {string} language Wiki language
+ * @returns {Promise<DataMap[]>} List of maps on the wiki
+ */
+export function getAllMaps(language = 'en') {
+    return httpGet(getApiUrl(language), {
+        action: 'query',
+        generator: 'allpages',
+        gapnamespace: '2900',
+        gapfilterredir: 'nonredirects',
+        gaplimit: 'max',
+        prop: 'revisions',
+        rvprop: 'ids|content',
+        rvslots: 'main',
+        format: 'json',
+        formatversion: '2',
+    }).then(data => data.query.pages
+        .filter((/** @type {any} */ page) =>
+            page.revisions &&
+            page.revisions.length > 0 &&
+            page.revisions[0].slots &&
+            page.revisions[0].slots.main &&
+            page.revisions[0].slots.main.contentmodel === 'datamap'
+        )
+        .map((/** @type {any} */ page) => {
+            const {slots, revid} = page.revisions[0];
+            const /** @type {DataMap} */ datamap = JSON.parse(slots.main.content);
+            datamap.custom = datamap.custom || new MetadataImpl();
+            datamap.custom.interwiki = datamap.custom.interwiki || {};
+            datamap.custom.interwiki[language] = new InterwikiDataImpl({
+                mapName: page.title.split(':').slice(1).join(':'),
+            });
+            datamap.custom.interwiki[language].revision = revid;
+            return datamap;
+        })
+        .filter((/** @type {DataMap} */ datamap) => !datamap.$fragment)
+    );
+}
+
+/**
+ * Returns the URLs of the given map files on the wiki.
+ * @param {string[]} filenames Map file names
+ * @param {string} language Wiki language
+ * @returns {Promise<string[]>} URLs of the given map files on the wiki
+ */
+export function getFileUrls(filenames, language = 'en') {
+    return httpGet(getApiUrl(language), {
+        action: 'query',
+        titles: filenames.map(name => `File:${name}`).join('|'),
+        prop: 'imageinfo',
+        iiprop: 'url',
+        format: 'json',
+        formatversion: '2'
+    }).then(data => data.query.pages
+        .map((/** @type {any} **/ page) => page.imageinfo[0].url));
+}
+
+/**
+ * Downloads a file from a URL and returns it as an ArrayBuffer.
+ * @param {string} url URL to download the file from
+ * @returns {Promise<ArrayBuffer>} Downloaded file data
+ */
+export function downloadFile(url) {
+    return new Promise((resolve, reject) => {
+        const xhr = new XMLHttpRequest();
+        xhr.open('GET', url, true);
+        xhr.responseType = 'arraybuffer';
+        xhr.onreadystatechange = readyStateChange(xhr, resolve, reject, true);
+        xhr.setRequestHeader('User-Agent', USER_AGENT);
+        xhr.send();
+    });
+}

extensions/includes/format.mjs

@@ -6,7 +6,13 @@ import {
     getColorProperty,
     getListProperty,
     getNumberProperty,
-    getStringProperty
+    getStringProperty,
+    getTiledColor,
+    isBoxOverlay,
+    isImageBackground,
+    isPolylineOverlay,
+    validateTiledPoint,
+    validateTiledRectangle
 } from './util.mjs';
 
 /**
@@ -221,6 +227,97 @@ export function convertTiledToDataMaps(map, mapName, language = 'en') {
     return datamap;
 }
 
+const TILE_SIZE = 32;
+
+/**
+ * Converts a DataMaps map to the Tiled map format.
+ * @param {DataMap} datamap DataMap to convert
+ * @returns {TileMap} Converted Tiled map object
+ */
+export function convertDataMapsToTiled(datamap) {
+    const primaryBg = datamap.backgrounds.find(isImageBackground);
+    if (!primaryBg) {
+        throw new Error('At least one background with an image is required to convert to Tiled format');
+    }
+    const crsBR = validateTiledPoint(datamap.crs.bottomRight);
+    const mapWidth = Math.ceil(crsBR[0] / TILE_SIZE);
+    const mapHeight = Math.ceil(crsBR[1] / TILE_SIZE);
+    const map = new TileMap();
+    map.tileWidth = TILE_SIZE;
+    map.tileHeight = TILE_SIZE;
+    map.width = mapWidth;
+    map.height = mapHeight;
+    if (datamap.disclaimer) {
+        map.setProperty('disclaimer', datamap.disclaimer);
+    }
+    if (datamap.settings && datamap.settings.leaflet) {
+        map.setProperty('popzoom', datamap.settings.leaflet.uriPopupZoom);
+    }
+    if (datamap.include) {
+        map.setProperty('include', datamap.include.join('\n'));
+    }
+    const /** @type {[string, ImageLayer][]} */ imageFiles = [];
+    for (const bg of datamap.backgrounds.filter(isImageBackground)) {
+        const layer = new ImageLayer();
+        // TODO: Support custom image property
+        layer.name = bg.image.replace(/\.png$/, '');
+        imageFiles.push([bg.image, layer]);
+        map.addLayer(layer);
+    }
+    let annotationLayer = new ObjectGroup('annotations');
+    map.addLayer(annotationLayer);
+    for (const overlay of (primaryBg.overlays || [])) {
+        const obj = new MapObject(overlay.name);
+        if (isBoxOverlay(overlay)) {
+            obj.shape = MapObject.Rectangle;
+            const [[x1, y1], [x2, y2]] = validateTiledRectangle(overlay.at);
+            obj.pos = { x: x1, y: y1 };
+            obj.width = x2 - x1;
+            obj.height = y2 - y1;
+            if (overlay.color) {
+                obj.setProperty('fill', getTiledColor(overlay.color));
+            }
+            if (overlay.borderColor) {
+                obj.setProperty('border', getTiledColor(overlay.borderColor));
+            }
+        } else if (isPolylineOverlay(overlay)) {
+            obj.polygon = overlay.path
+                .map(p => validateTiledPoint(p))
+                .map(p => ({ x: p[1], y: p[0] }));
+            obj.pos = { x: 0, y: 0 };
+            obj.shape = MapObject.Polyline;
+            if (overlay.color) {
+                obj.setProperty('color', getTiledColor(overlay.color));
+            }
+            if (overlay.thickness) {
+                obj.setProperty('thickness', overlay.thickness);
+            }
+        }
+        annotationLayer.addObject(obj);
+    }
+    // TODO: Support nested layers
+    for (const [layerName, markers] of Object.entries(datamap.markers).reverse()) {
+        const layer = new ObjectGroup(layerName);
+        for (const m of markers) {
+            const obj = new MapObject(m.name);
+            obj.pos = {
+                x: m.x,
+                y: m.y
+            };
+            obj.shape = MapObject.Point;
+            obj.setProperties({
+                page: m.article,
+                description: m.description,
+                image: m.image,
+                plain: m.isWikitext === undefined ? undefined : !m.isWikitext
+            });
+            layer.addObject(obj);
+        }
+        map.addLayer(layer);
+    }
+    return map;
+}
+
 /**
  * Converts a Tiled map to DataMaps format.
  * @param {TileMap} map Tiled map to convert to DataMaps
@@ -235,8 +332,22 @@ function write(map, filePath) {
     file.commit();
 }
 
+/**
+ * Converts a DataMaps map to the Tiled map format.
+ * @param {string} filePath Path to the file with the DataMaps map
+ * @returns {TileMap} Tiled map
+ */
+function read(filePath) {
+    const file = new TextFile(filePath, TextFile.ReadOnly);
+    const content = file.readAll();
+    file.close();
+    const datamap = JSON.parse(content);
+    return convertDataMapsToTiled(datamap.en);
+}
+
 export default /** @type {ScriptedMapFormat} */ {
     extension: 'mw-datamaps',
     name: 'DataMaps',
+    read,
     write
 };

extensions/includes/pull.mjs

@@ -0,0 +1,57 @@
+import { downloadFile, getAllMaps, getFileUrls } from "./api.mjs";
+import { getLanguageCodes } from "./language.mjs";
+import { isImageBackground } from "./util.mjs";
+
+export default function run() {
+    const languages = getLanguageCodes();
+    Promise.all(languages
+        .map(language => getAllMaps(language)))
+        .then(maps => {
+            /** @type {Record<string, Record<string, DataMap>>} */
+            const allMaps = {};
+            const /** @type {string[]} */ backgrounds = [];
+            for (const [index, language] of languages.entries()) {
+                for (const map of maps[index]) {
+                    const mapName = map.custom?.interwiki?.en?.mapName;
+                    if (!mapName) {
+                        tiled.log(`Map "${map.custom?.interwiki?.[language].mapName}" is missing an English map name in its metadata, skipping it.`);
+                        continue;
+                    }
+                    allMaps[mapName] = allMaps[mapName] || {};
+                    allMaps[mapName][language] = map;
+                    backgrounds.push(...map.backgrounds
+                        .filter(isImageBackground)
+                        .map(bg => bg.image));
+                }
+            }
+            const baseDir = FileInfo.path(tiled.projectFilePath);
+            for (const map of Object.values(allMaps)) {
+                const mapName = map.en.custom?.interwiki?.en.mapName;
+                if (!mapName) {
+                    continue;
+                }
+                const textFile = new TextFile(
+                    FileInfo.joinPaths(baseDir, `${mapName}.mw-datamaps`),
+                    TextFile.WriteOnly
+                );
+                textFile.write(JSON.stringify(map));
+                textFile.commit();
+            }
+            const backgroundFiles = [...new Set(backgrounds)];
+            return getFileUrls(backgroundFiles, 'en')
+                .then(urls => Promise.all(urls.map(url => downloadFile(url))))
+                .then(files => files.forEach((file, index) => {
+                    const filename = backgroundFiles[index];
+                    const binaryFile = new BinaryFile(
+                        FileInfo.joinPaths(baseDir, 'images', filename),
+                        BinaryFile.WriteOnly
+                    );
+                    binaryFile.write(file);
+                    binaryFile.commit();
+                }));
+        })
+        .catch(error => {
+            tiled.alert(`An error occurred while pulling maps from the wiki: ${error.message}`);
+            tiled.log(`Error details: ${error.stack}`);
+        });
+}