dtMediaWiki.lua

--[[dtMediaWiki is a darktable plugin which exports images to Wikimedia Commons
    Author: Trougnouf (Benoit Brummer) <trougnouf@gmail.com>
    Contributor: Simon Legner (simon04)

Dependencies:
* lua-sec: Lua bindings for OpenSSL library to provide TLS/SSL communication
* lua-luajson: JSON parser/encoder for Lua
* lua-multipart-post: HTTP Multipart Post helper
]]
local dt = require "darktable"
local MediaWikiApi = require "contrib/dtMediaWiki/lib/mediawikiapi"
local version = 91

--[[The version number is generated by .git/hooks/pre-commit (+x)
    with the following content:
  #!/bin/sh
  NEWVERSION="$(expr "$(git log master --pretty=oneline | wc -l)" + 1)"
  sed -i -r "s/local version = [0-9]+/local version = ${NEWVERSION}/1" dtMediaWiki.lua
  git add dtMediaWiki.lua
  ]]

-- Use this _ function for translatable strings
local gettext = dt.gettext
gettext.bindtextdomain("dtMediaWiki", dt.configuration.config_dir .. "/lua/locale/")
local function translate(msgid)
  return gettext.dgettext("dtMediaWiki", msgid)
end

local function msgout(str)
  -- Use msgout(gettext.dgettext("dtMediaWiki", "STRING")) to output STRING in both the terminal and GUI
  print(str)
  dt.print(str)
end

local function dbgout(str)
  -- Use dbgout(gettext.dgettext("dtMediaWiki", "STRING")) to output AMESSAGE in the debug terminal (darktable -d lua)
  dt.print_log(str)
end
-- Preference entries
local preferences_prefix = "mediawiki"

dt.preferences.register(
  preferences_prefix,
  "cat_cam",
  "bool",
  translate("Commons: Categorize camera?"),
  translate("A category will be added with the camera information " ..
    "(eg: [[Category:Taken with Fujifilm X-E2 and XF18-55mmF2.8-4 R LM OIS]])"),
  false
)

dt.preferences.register(
  preferences_prefix,
  "desc_templates",
  "string",
  translate("Commons: Templates to be placed in {{Information |description= ...}}"),
  translate('These templates are placed in the {{Information |description= ...}} field. (comma-separated)'),
  "Description,Depicted person,en,de,fr,es,ja,ru,zh,it,pt,ar"
)

dt.preferences.register(
  preferences_prefix,
  "overwrite",
  "bool",
  translate("Commons: Overwrite existing images?"),
  translate("Existing images will be overwritten  without confirmation, otherwise the upload will fail."),
  false
)

dt.preferences.register(
  preferences_prefix,
  "authorpattern",
  "string",
  translate("Commons: Preferred author pattern"),
  translate("Determines the author value; variables are $USERNAME, $CREATOR"),
  "[[User:$USERNAME|$CREATOR]]"
)

dt.preferences.register(
  preferences_prefix,
  "namepattern",
  "string",
  translate("Commons: Preferred filename pattern"),
  translate("Determines the filename value; variables are $TITLE, $DESCRIPTION, $FILE_NAME"),
  "$TITLE ($FILE_NAME)"
)

-- Preference for allowing empty Information|description field on Commons
dt.preferences.register(
  preferences_prefix,
  "allow_empty_description",
  "bool",
  translate("Commons: Allow empty 'Information|description' on page?"),
  translate("If unchecked (default), images where both title and description metadata are empty will not be uploaded, as this results in an empty '{{Information|description=...}}' field on Wikimedia Commons. If checked, such uploads are allowed (unless the filename pattern specifically requires title or description)."),
  false
)

-- FIXME: can't figure enum out so using a text field in the meantime
-- see also: https://docs.darktable.org/lua/stable/lua.api.manual/darktable/darktable.preferences/#darktablepreferencesregister
dt.preferences.register(
  preferences_prefix,
  "default_license",
  "string",
  translate("Commons: Default license"),
  translate("Default license for images uploaded to Wikimedia Commons (eg: CC-BY-SA-4.0, CC-BY-4.0, PD, ...)"),
  "CC-BY-SA-4.0"
)

dt.preferences.register(
  preferences_prefix,
  "password",
  "string", -- TODO Use Lua password storage once in release. See https://github.com/darktable-org/darktable/pull/7508
  translate("Wikimedia password"),
  translate("Wikimedia Commons password (to be stored in plain-text!)"),
  ""
)

dt.preferences.register(
  preferences_prefix,
  "username",
  "string",
  translate('Wikimedia username'),
  translate("Wikimedia Commons username"),
  ""
)

local namepattern_default = dt.preferences.read(preferences_prefix, "namepattern", "string")

local namepattern_widget =
    dt.new_widget("entry") {
      tooltip = table.concat(
        {
          translate("Determines the `File:` page name on Wikimedia Commons."),
          translate("Recognized variables:"),
          translate("  $FILE_NAME - basename of the original input image (e.g., 'IMG_1234')"),
          translate("  $TITLE - title from image metadata"),
          translate("  $DESCRIPTION - description from image metadata"),
          translate("Note: Validation checks during export initialization will ensure required fields are populated based on this pattern and relevant preferences.")
        },
        "\n"
      ),
      text = dt.preferences.read(preferences_prefix, "namepattern", "string"),
      reset_callback = function(self)
        self.text = namepattern_default
        dt.preferences.write(preferences_prefix, "namepattern", "string", self.text)
      end
    }

-- language widget shown in lighttable export
local language_widget =
    dt.new_widget("entry") {
      text = "en",
      tooltip = translate("Description language code. Additional descriptions may be added with tag "
        .. "{{Description|language_code|description_text}} or any of the templates listed in "
        .. "the desc_template setting."),
      reset_callback = function(self)
        self.text = "en"
      end
    }

-- Generate image name
local function make_image_name(image, tmp_exp_path)
  local basename = image.filename:match "[^.]+"
  -- Read current pattern, falling back to default if widget text is empty or preference is missing
  local outname_pattern = namepattern_widget.text
  if outname_pattern == nil or outname_pattern == '' then
    outname_pattern = namepattern_default
  end

  local final_outname = outname_pattern
  final_outname = final_outname:gsub("%$FILE_NAME", basename)
  final_outname = final_outname:gsub("%$TITLE", image.title or "")
  final_outname = final_outname:gsub("%$DESCRIPTION", image.description or "")
  
  local name_part_check = final_outname
  if name_part_check:match("^%s*$") or name_part_check == "" then
      dt.print_log(translate("Filename pattern resulted in an empty name (before extension), falling back to input image basename."))
      final_outname = basename
  end
  
  local ext = tmp_exp_path:match "[^.]+$"
  return final_outname .. "." .. ext
end

-- Round to 1 decimal, remove useless .0's and convert number to string. Ensure that "." decimal separator is used
local function fmt_flt(num)
  return string.format("%.1f", num):gsub(",", "."):gsub("%.0", "")
end

-- Get description field from the description (or title) metadata
local function get_description(image)
  if image.description ~= "" then
    return image.description
  else
    return image.title -- Fallback to title if description is empty
  end
end

local function split(astring) 
  local sep, fields = ",", {}
  local pattern = string.format("([^%s]+)", sep)
  astring:gsub(pattern, function(c) fields[#fields + 1] = c end)
  return fields
end

-- get description templates which needs to be added to {{Information| description=...}}
local function get_intl_descriptions(image, discarded_tags)
  local desc_templates = split(dt.preferences.read(preferences_prefix, "desc_templates", "string"))
  local intl_descriptions = ""
  for _, tag in pairs(dt.tags.get_tags(image)) do
    if tag ~= 0 then -- workaround for dt bug #9715
      local tagstr = tag.name
      for _, dtemplate in pairs(desc_templates) do
        if tagstr:sub(1, #dtemplate + 3) == '{{' .. dtemplate .. '|' then
          intl_descriptions = intl_descriptions .. tagstr
          discarded_tags[tagstr] = true
        end
      end
    end
  end
  return intl_descriptions
end


-- Get other fields that are then added to the Information template (TODO document this feature)
local function get_other_fields(image, discarded_tags)
  local other_fields = ''
  for _, tag in pairs(dt.tags.get_tags(image)) do
    if tag ~= 0 then -- workaround for dt bug #9715
      local tagstr = tag.name
      if string.sub(tagstr, 0, 20) == '{{Information field|'
          or string.sub(tagstr, 0, 7) == '{{InFi|' then
        other_fields = other_fields .. tagstr
        discarded_tags[tagstr] = true
      end
    end
  end
  return other_fields
end

local function substitute_keywords(string, image)
  local username = dt.preferences.read(preferences_prefix, "username", "string")
  string = string:gsub("$USERNAME", username)
  string = string:gsub("$CREATOR", image.creator or username)
  string = string:gsub("$FILE_NAME", image.filename)
  string = string:gsub("$DATETIME", image.exif_datetime_taken)
  string = string:gsub("$YEAR", image.exif_datetime_taken:sub(0, 4))
  string = string:gsub("\\comma", ",")
  return string
end

-- get "Other versions" of this image, ie different views of the scene or duplicate
-- of this image (the latter needs to have a different title).
-- Used with the "alt:<fn>" tag;
-- matches <fn> contained in other images' filename in the current library.
-- input:
---- image: working image (dt_lua_image_t)
---- tmp_img_fn: working image temporary export filename (string)
-- output:
---- gallery following "Other versions = " on the image page, or an empty string (string)
-- TODO document this feature
local function get_alt_images(image, tmp_img_fn)
  local alts = {}
  for _, tag in pairs(dt.tags.get_tags(image)) do
    if tag ~= 0 then -- workaround for dt bug #9715
      if string.sub(tag.name, 1, 4) == "alt:" then
        table.insert(alts, string.sub(tag.name, 5))
      end
    end
  end
  if next(alts) == nil then
    return ""
  end
  local altcode = { "<gallery showfilename=yes>" }
  for _, img in pairs(dt.collection) do
    dbgout(gettext.dgettext("dtMediaWiki", ("dtMediaWiki.get_alt_images: looking at ")) .. img.filename)
    for _, altimg in pairs(alts) do
      dbgout(gettext.dgettext("dtMediaWiki", ("dtMediaWiki.get_alt_images: looking for ")) .. altimg)
      if string.find(img.filename, altimg) then
        local alt_fn = make_image_name(img, tmp_img_fn)
        table.insert(altcode, alt_fn)
        dbgout(gettext.dgettext("dtMediaWiki", ("dtMediaWiki.get_alt_images: found ")) .. alt_fn)
      end
    end
  end
  table.insert(altcode, '</gallery>')
  return table.concat(altcode, "\n")
end

-- helper to get the license from image rights or default preference
local function get_license(image)
  local license = image.rights
  if license == "" then
    license = dt.preferences.read(preferences_prefix, "default_license", "string")
  end
  return license
end

-- Generate an image page with all required info from tags, metadata, and such.
local function make_image_page(image, tmp_img_fn)
  local discarded_tags = {}
  local imgpg = { "=={{int:filedesc}}==\n{{Information" }
  table.insert(imgpg, "|description={{" .. language_widget.text .. "|1="
    .. substitute_keywords(get_description(image), image) .. "}}"
    .. get_intl_descriptions(image, discarded_tags))
  local date = image.exif_datetime_taken
  date = date:gsub("(%d%d%d%d):(%d%d):(%d%d)", "%1-%2-%3") -- format date in ISO 8601 / RFC 3339
  table.insert(imgpg, "|date=" .. date)
  table.insert(imgpg, "|source={{own}}")
  local author = dt.preferences.read(preferences_prefix, "authorpattern", "string")
  author = substitute_keywords(author, image)
  table.insert(imgpg, "|author=" .. author)
  table.insert(imgpg, '|other fields = ' .. get_other_fields(image, discarded_tags))
  table.insert(imgpg, '|other versions = ' .. get_alt_images(image, tmp_img_fn))
  table.insert(imgpg, "}}")
  if image.latitude ~= nil and image.longitude ~= nil then
    table.insert(imgpg, "{{Location |1=" .. string.gsub(image.latitude, ",", ".")
      .. " |2=" .. string.gsub(image.longitude, ",", ".") .. " }}")
  end
  table.insert(imgpg, "=={{int:license-header}}==")
  table.insert(imgpg, "{{self|" .. get_license(image) .. "}}")
  for _, tag in pairs(dt.tags.get_tags(image)) do
    if tag ~= 0 then -- workaround for dt bug #9715
      local subbed_tag = substitute_keywords(tag.name, image)
      if string.sub(subbed_tag, 1, 9) == "Category:" then
        table.insert(imgpg, "[[" .. subbed_tag .. "]]")
      elseif subbed_tag:sub(1, 2) == "{{" and not discarded_tags[subbed_tag] then
        table.insert(imgpg, subbed_tag)
      end
    end
  end
  if dt.preferences.read(preferences_prefix, "cat_cam", "bool") then
    if image.exif_model ~= "" then
      local model = image.exif_maker:sub(1, 1) .. image.exif_maker:sub(2):lower()
      local catcam = "[[Category:Taken with " .. model .. " " .. image.exif_model
      if image.exif_lens ~= "" then
        catcam = catcam .. " and " .. image.exif_lens .. "]]"
      else
        catcam = catcam .. "]]"
      end
      table.insert(imgpg, catcam)
    end
    if image.exif_aperture then
      -- convert aperture to US_en locale and remove trailing .0 if there is any
      table.insert(imgpg, "[[Category:F-number f/" .. fmt_flt(image.exif_aperture) .. "]]")
    end
    if image.exif_focal_length ~= "" then
      table.insert(imgpg, "[[Category:Lens focal length " .. fmt_flt(image.exif_focal_length) .. " mm]]")
    end
    if image.exif_iso ~= "" then
      table.insert(imgpg, "[[Category:ISO speed rating " .. fmt_flt(image.exif_iso) .. "]]")
    end
  end
  table.insert(imgpg, "[[Category:Uploaded with dtMediaWiki]]")
  imgpg = table.concat(imgpg, "\n")
  return imgpg
end

-- comment widget shown in lighttable export
local comment_widget =
    dt.new_widget("entry") {
      text = translate("Uploaded with dtMediaWiki ") .. version,
      reset_callback = function(self)
        self.text = translate("Uploaded with dtMediaWiki ") .. version
      end
    }

-- check if extension is supported (when using the copy mode we only get the extension at this point)
local function _register_storage_store_final_format_check(fn)
  local extension = fn:match("[^.]+$"):lower()
  if extension ~= "jpg" and extension ~= "png" and extension ~= "tif" and extension ~= "webp" then
    -- TODO would be nice to assert that copy mode is used
    -- TODO add avif when this ticket will be solved https://phabricator.wikimedia.org/T257719
    msgout(gettext.dgettext("dtMediaWiki",
      ("dtMediaWiki._register_storage_store_final_format_check warning: using copy mode from an unsupported image format is known to cause a crash.")))
    msgout(gettext.dgettext("dtMediaWiki", ("Error: ")) .. fn ..
      gettext.dgettext("dtMediaWiki",
        (" has an unsupported extension, won't be exported to Wikimedia Commons in copy mode")))
    return false
  end
  return true
end

--This function is called once for each exported image
local function register_storage_store(_, image, _, tmp_exp_path, _, _, _, extra_data)
  msgout(gettext.dgettext("dtMediaWiki", ("register_storage_store: exporting the following image:")))
  msgout(tmp_exp_path .. '(' .. image.title .. ')')
  for _, tag in pairs(dt.tags.get_tags(image)) do
    if tag == 0 then
      msgout(gettext.dgettext("dtMediaWiki", ("BUG: invalid tag 0 for image ")) .. image.title)
    end
  end
  local imagepage = make_image_page(image, tmp_exp_path)
  local imagename = make_image_name(image, tmp_exp_path)
  --print(imagepage)
  local success = _register_storage_store_final_format_check(tmp_exp_path)
  if success then
    success = MediaWikiApi.uploadfile(
      tmp_exp_path,
      imagepage,
      imagename,
      dt.preferences.read(preferences_prefix, "overwrite", "bool"),
      comment_widget.text
    )
  end
  if success then
    msgout(gettext.dgettext("dtMediaWiki", ("exported ")) .. imagename) -- that is the path also
  else
    msgout(gettext.dgettext("dtMediaWiki", ("Failed to export ")) .. imagename)
    extra_data["failures_count"] = extra_data["failures_count"] + 1
  end
end

--This function is called once all images are processed and all store calls are finished.
local function register_storage_finalize(_, image_table, extra_data)
  local successfully_initialized_count = 0
  for _ in pairs(image_table) do 
    successfully_initialized_count = successfully_initialized_count + 1
  end
  
  local successfully_uploaded_count = successfully_initialized_count - extra_data["failures_count"]

  msgout(gettext.dgettext("dtMediaWiki", ("exported ")) .. successfully_uploaded_count .. "/" .. extra_data["init_img_cnt"] ..
    gettext.dgettext("dtMediaWiki", (" images to Wikimedia Commons")))
end

--A function called to check if a given image format is supported by the Lua storage;
--This is used to build the dropdown format list for the GUI.
local function register_storage_supported(_, format)
  local ext = format.extension
  return ext == "jpg" or ext == "png" or ext == "tif" or ext == "webp" or ext == "" -- "" is for the "copy" mode
end

--A function called before storage happens
--This function can change the list of exported functions
local function register_storage_initialize(_, _, images, _, extra_data)
  local out_images = {}
  extra_data["failures_count"] = 0 -- For store() failures
  extra_data["init_img_cnt"] = #images
  local init_failures_count = 0 -- For initialize() failures

  -- This preference means "Allow the {{Information|description=...}} field on Commons to be empty"
  -- which happens if both image.title and image.description are empty.
  local allow_empty_page_content_pref = dt.preferences.read(preferences_prefix, "allow_empty_description", "bool")
  local current_namepattern = namepattern_widget.text
  if current_namepattern == nil or current_namepattern == '' then
    current_namepattern = namepattern_default
  end

  for _, img in pairs(images) do
    local is_valid_for_export = true
    local failure_reasons = {}

    -- BUG? images contain "0" value (with key 3 in tested instance) instead of dt_lua_image_t
    if type(img) == 'number' then
      is_valid_for_export = false
      table.insert(failure_reasons, 
        gettext.dgettext("dtMediaWiki", ("BUG: dt.register_storage.initialize sent a number instead of dt_lua_image_t in images table: ")) .. img
      )
    else
      -- Reason 1: License
      if get_license(img) == "" then
        is_valid_for_export = false
        table.insert(failure_reasons, translate("has no rights"))
      end

      local title_empty = (img.title == nil or img.title == "")
      local description_empty = (img.description == nil or img.description == "")

      -- Reason 2: Title required by filename pattern but missing
      if current_namepattern:find("%$TITLE") and title_empty then
        is_valid_for_export = false
        table.insert(failure_reasons, translate("is missing a title, which is required by the filename pattern"))
      end

      -- Reason 3: Description required by filename pattern but missing
      if current_namepattern:find("%$DESCRIPTION") and description_empty then
        is_valid_for_export = false
        table.insert(failure_reasons, translate("is missing a description, which is required by the filename pattern"))
      end
      
      -- Reason 4: The {{Information|description=...}} field on Commons would be empty,
      -- and the preference 'allow_empty_description' (which means allow empty page content) is false.
      -- This happens if both title and description metadata are empty.
      if not allow_empty_page_content_pref and title_empty and description_empty then
         is_valid_for_export = false
         table.insert(failure_reasons, translate("is missing both title and description, resulting in an empty 'Information|description' field on the Commons page (and the 'Allow empty Information|description...' preference is off)"))
      end
    end

    if not is_valid_for_export then
      init_failures_count = init_failures_count + 1
      local error_intro = ""
      if type(img) == 'number' then
         error_intro = gettext.dgettext("dtMediaWiki", ("Error: "))
      else
         error_intro = gettext.dgettext("dtMediaWiki", ("Error: ")) .. (img.path or "Unknown image") .. " "
      end
      
      local unique_reasons = {}
      local seen_reasons_map = {}
      for _, reason in ipairs(failure_reasons) do
        if not seen_reasons_map[reason] then
          table.insert(unique_reasons, reason)
          seen_reasons_map[reason] = true
        end
      end

      if #unique_reasons > 0 then
        msgout(
          error_intro .. table.concat(unique_reasons, "; ") ..
          gettext.dgettext("dtMediaWiki", (", won't be exported to Wikimedia Commons"))
        )
      end
    else
      table.insert(out_images, img)
    end
  end
  return out_images
end

-- widgets shown in lighttable
local export_widgets =
    dt.new_widget("box") {
      dt.new_widget("box") {
        orientation = "horizontal",
        dt.new_widget("label") { label = translate("Naming pattern:") },
        namepattern_widget
      },
      dt.new_widget("box") {
        orientation = "horizontal",
        dt.new_widget("label") { label = translate("Comment:") },
        comment_widget
      },
      dt.new_widget("box") {
        orientation = "horizontal",
        dt.new_widget("label") { label = translate("Language code:"),
          tooltip = translate("Description language code (eg: en, fr, ...).")
              .. translate("More descriptions can be entered in the tags (eg: {{fr|Une description}}") },
        language_widget
      }
    }

-- Darktable target storage entry
if
    MediaWikiApi.login(
      dt.preferences.read(preferences_prefix, "username", "string"),
      dt.preferences.read(preferences_prefix, "password", "string")
    )
then
  -- https://docs.darktable.org/lua/stable/lua.api.manual/darktable/darktable.register_storage
  dt.register_storage(
    "mediawiki",
    "Wikimedia Commons",
    register_storage_store,
    register_storage_finalize,
    register_storage_supported,
    register_storage_initialize,
    export_widgets
  )
else
  msgout(translate("Unable to log into Wikimedia Commons, export disabled."))
end