diff --git a/src/common/exif.cc b/src/common/exif.cc
index 0a9aa5f80e70..5c5f170b2611 100644
--- a/src/common/exif.cc
+++ b/src/common/exif.cc
@@ -610,7 +610,6 @@ static bool _exif_decode_xmp_data(dt_image_t *img,
     if(version == -1 || version > 0)
     {
       dt_pthread_mutex_lock(&darktable.metadata_threadsafe);
-      if(!exif_read) dt_metadata_clear(imgs, FALSE);
 
       for(GList *iter = dt_metadata_get_list(); iter; iter = iter->next)
       {
@@ -629,6 +628,12 @@ static bool _exif_decode_xmp_data(dt_image_t *img,
           dt_metadata_set_import(img->id, metadata->tagname, value);
           free(adr);
         }
+        else if(!exif_read && strstr(metadata->tagname, "Xmp.") == metadata->tagname)
+        {
+          // Only remove Xmp. metadata fields, do not touch metadata from other
+          // sources (e.g. Exif.*).
+          dt_metadata_unset(img->id, metadata->tagname, FALSE);
+        }
       }
       dt_pthread_mutex_unlock(&darktable.metadata_threadsafe);
     }
@@ -2352,6 +2357,52 @@ static bool _exif_decode_exif_data(dt_image_t *img, Exiv2::ExifData &exifData)
       }
     };
 
+    dt_pthread_mutex_lock(&darktable.metadata_threadsafe);
+    for(GList *iter = dt_metadata_get_list(); iter; iter = iter->next)
+    {
+      dt_metadata_t *metadata = (dt_metadata_t *)iter->data;
+      if(!FIND_EXIF_TAG(metadata->tagname))
+      {
+        continue;
+      }
+
+      int ival = pos->toLong();
+      std::string str = pos->print(&exifData);
+      char *value = g_locale_to_utf8(str.c_str(), str.length(), NULL, NULL, NULL);
+      if(value == NULL)
+      {
+        // need non-const char* for g_strstrip
+        value = g_strdup(str.c_str());
+      }
+      g_strstrip(value);
+
+      gchar *str_value = g_strdup_printf("(%d)", ival);
+      if(g_strcmp0(value, str_value) == 0)
+      {
+        // no string mapping available in exiv2, so we use exiv2's
+        // default string conversion.
+        g_free(value);
+        str = pos->toString();
+        // for consistency with the handling above. don't want to mix two
+        // allocators, that causes me headaches.
+        value = g_strdup(str.c_str());
+        // no need to keep this around for longer.
+        str = nullptr;
+      }
+      g_free(str_value);
+
+      char *adr = value;
+      // Skip any lang="" or charset=xxx
+      while(!strncmp(value, "lang=", 5) || !strncmp(value, "charset=", 8))
+      {
+        while(*value != ' ' && *value) value++;
+        while(*value == ' ') value++;
+      }
+      dt_metadata_set_import(img->id, metadata->tagname, value);
+      g_free(adr);
+    }
+    dt_pthread_mutex_unlock(&darktable.metadata_threadsafe);
+
     img->exif_inited = TRUE;
     return true;
   }
diff --git a/src/common/metadata.c b/src/common/metadata.c
index 0e5f2cab9838..49c47e011e7e 100644
--- a/src/common/metadata.c
+++ b/src/common/metadata.c
@@ -838,6 +838,34 @@ void dt_metadata_set_list_id(const GList *img,
   }
 }
 
+void dt_metadata_unset(const dt_imgid_t imgid, const char *key, const gboolean undo_on)
+{
+  if(!key || !dt_is_valid_imgid(imgid)) return;
+
+  int keyid = dt_metadata_get_keyid(key);
+  if(keyid == -1) return;
+
+  GList *imgs = NULL;
+  imgs = g_list_prepend(imgs, GINT_TO_POINTER(imgid));
+  GList *undo = NULL;
+  if(undo_on) dt_undo_start_group(darktable.undo, DT_UNDO_METADATA);
+
+  const gchar *ckey = g_strdup_printf("%d", keyid);
+  GList *metadata = NULL;
+  metadata = g_list_append(metadata, (gpointer)ckey);
+  metadata = g_list_append(metadata, NULL);
+
+  _metadata_execute(imgs, metadata, &undo, undo_on, DT_MA_REMOVE);
+
+  g_list_free_full(metadata, g_free);
+  g_list_free(imgs);
+  if(undo_on)
+  {
+    dt_undo_record(darktable.undo, NULL, DT_UNDO_METADATA, undo, _pop_undo, _metadata_undo_data_free);
+    dt_undo_end_group(darktable.undo);
+  }
+}
+
 gboolean dt_metadata_already_imported(const char *filename, const char *datetime)
 {
   if(!filename || !datetime)
diff --git a/src/common/metadata.h b/src/common/metadata.h
index c2f6b3c4e297..33453218c504 100644
--- a/src/common/metadata.h
+++ b/src/common/metadata.h
@@ -98,6 +98,8 @@ void dt_metadata_set_list(const GList *imgs, GList *key_value, const gboolean un
     if clear_on TRUE the image metadata are cleared before attaching the new ones*/
 void dt_metadata_set_list_id(const GList *img, const GList *metadata, const gboolean clear_on,
                              const gboolean undo_on);
+/** Unset a specific metadata key for a specific image. Noop if the key isn't set. */
+void dt_metadata_unset(const dt_imgid_t imgid, const char *key, const gboolean undo_on);
 /** Get metadata (named keys) for a specific image, or all selected for an invalid imgid
     For keys which return a string, the caller has to make sure that it
     is freed after usage. With mutex lock. */