Updated Onvista PDF Extractor to support parsing of ex-date of dividends

Issue: #5439

Author: ZfT2

name.abuchen.portfolio.tests/src/name/abuchen/portfolio/datatransfer/ExtractorMatchers.java

@@ -396,6 +396,17 @@ public static Matcher<Transaction> hasDate(String dateString)
                         Transaction::getDateTime);
     }
 
+    public static Matcher<Transaction> hasExDate(String dateString)
+    {
+        var expected = dateString.contains("T") //$NON-NLS-1$
+                        ? LocalDateTime.parse(dateString)
+                        : LocalDate.parse(dateString).atStartOfDay();
+
+        return new PropertyMatcher<>("exDate", //$NON-NLS-1$
+                        expected, //
+                        t -> ((AccountTransaction) t).getExDate());
+    }
+
     public static Matcher<Transaction> hasShares(double value)
     {
         // work with BigDecimal to have better assertion failed messages

name.abuchen.portfolio.tests/src/name/abuchen/portfolio/datatransfer/pdf/onvista/OnvistaPDFExtractorTest.java

@@ -8,6 +8,7 @@
 import static name.abuchen.portfolio.datatransfer.ExtractorMatchers.hasAmount;
 import static name.abuchen.portfolio.datatransfer.ExtractorMatchers.hasCurrencyCode;
 import static name.abuchen.portfolio.datatransfer.ExtractorMatchers.hasDate;
+import static name.abuchen.portfolio.datatransfer.ExtractorMatchers.hasExDate;
 import static name.abuchen.portfolio.datatransfer.ExtractorMatchers.hasFees;
 import static name.abuchen.portfolio.datatransfer.ExtractorMatchers.hasForexGrossValue;
 import static name.abuchen.portfolio.datatransfer.ExtractorMatchers.hasGrossValue;
@@ -37,6 +38,7 @@
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.collection.IsEmptyCollection.empty;
 import static org.junit.Assert.assertNull;
 
@@ -2053,6 +2055,7 @@ public void testDividende01()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2016-04-21T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2016-04-21T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(50)));
         assertThat(transaction.getSource(), is("Dividende01.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 77110599"));
@@ -2113,6 +2116,7 @@ public void testDividende02()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2016-12-16T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2016-12-14T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(1.0545)));
         assertThat(transaction.getSource(), is("Dividende02.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 55746925 | Ertrag für 2016/17"));
@@ -2129,6 +2133,7 @@ public void testDividende02()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2016-12-16T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2016-12-14T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(1.2879)));
         assertThat(transaction.getSource(), is("Dividende02.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 97603916 | Ertrag für 2016/17"));
@@ -2145,6 +2150,7 @@ public void testDividende02()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2016-12-16T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2016-12-14T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(9.9225)));
         assertThat(transaction.getSource(), is("Dividende02.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 33071326 | Ertrag für 2016"));
@@ -2189,6 +2195,7 @@ public void testDividende03()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2019-07-15T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2019-06-27T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(100)));
         assertThat(transaction.getSource(), is("Dividende03.txt"));
         assertNull(transaction.getNote());
@@ -2234,6 +2241,7 @@ public void testDividende03WithSecurityInEUR()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2019-07-15T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2019-06-27T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(100)));
         assertThat(transaction.getSource(), is("Dividende03.txt"));
         assertNull(transaction.getNote());
@@ -2285,6 +2293,7 @@ public void testDividende04()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2019-03-26T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2019-03-22T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(6)));
         assertThat(transaction.getSource(), is("Dividende04.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 29013705"));
@@ -2329,6 +2338,7 @@ public void testDividende04WithSecurityInEUR()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2019-03-26T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2019-03-22T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(6)));
         assertThat(transaction.getSource(), is("Dividende04.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 29013705"));
@@ -2379,6 +2389,7 @@ public void testDividende05()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2020-02-14T00:00")));
+        assertNull(transaction.getExDate());
         assertThat(transaction.getShares(), is(Values.Share.factorize(50)));
         assertThat(transaction.getSource(), is("Dividende05.txt"));
         assertNull(transaction.getNote());
@@ -2424,6 +2435,7 @@ public void testDividende06()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2015-03-04T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2015-03-02T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(28)));
         assertThat(transaction.getSource(), is("Dividende06.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 96937413 | Ertrag für 2014"));
@@ -2469,6 +2481,7 @@ public void testDividende07()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2016-12-15T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2016-12-15T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(5.8192)));
         assertThat(transaction.getSource(), is("Dividende07.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 14053767 | Ertrag für 2016/17"));
@@ -2513,6 +2526,7 @@ public void testDividende08()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2019-02-05T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2019-01-31T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(32)));
         assertThat(transaction.getSource(), is("Dividende08.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 12345 | Ertrag für 2018"));
@@ -2557,6 +2571,7 @@ public void testDividende08WithSecurityInEUR()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2019-02-05T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2019-01-31T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(32)));
         assertThat(transaction.getSource(), is("Dividende08.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 12345 | Ertrag für 2018"));
@@ -2607,6 +2622,7 @@ public void testDividende09()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2019-09-11T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2019-08-08T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(40)));
         assertThat(transaction.getSource(), is("Dividende09.txt"));
         assertNull(transaction.getNote());
@@ -2652,6 +2668,7 @@ public void testDividende10()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2020-11-25T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2020-09-29T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(46)));
         assertThat(transaction.getSource(), is("Dividende10.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 11223344"));
@@ -2697,6 +2714,7 @@ public void testDividende10WithSecurityInEUR()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2020-11-25T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2020-09-29T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(46)));
         assertThat(transaction.getSource(), is("Dividende10.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 11223344"));
@@ -2748,6 +2766,7 @@ public void testDividende11()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2020-11-12T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2020-11-04T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(500)));
         assertThat(transaction.getSource(), is("Dividende11.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 23344420 | Ertrag für 2020"));
@@ -2793,6 +2812,7 @@ public void testDividende11WithSecurityInEUR()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2020-11-12T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2020-11-04T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(500)));
         assertThat(transaction.getSource(), is("Dividende11.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 23344420 | Ertrag für 2020"));
@@ -2844,6 +2864,7 @@ public void testDividende12()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2010-11-17T00:00")));
+        assertNull(transaction.getExDate());
         assertThat(transaction.getShares(), is(Values.Share.factorize(1)));
         assertThat(transaction.getSource(), is("Dividende12.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 63302459"));
@@ -2888,6 +2909,7 @@ public void testDividende13()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2021-09-15T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2021-09-15T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(549)));
         assertThat(transaction.getSource(), is("Dividende13.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 34091609 | Ertrag für 2021/22"));
@@ -2933,6 +2955,7 @@ public void testDividende14()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2021-12-17T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2021-12-02T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(1000)));
         assertThat(transaction.getSource(), is("Dividende14.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 59788848"));
@@ -2978,6 +3001,7 @@ public void testDividende14WithSecurityInEUR()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2021-12-17T00:00")));
+        assertThat(transaction.getExDate(), is(LocalDateTime.parse("2021-12-02T00:00")));
         assertThat(transaction.getShares(), is(Values.Share.factorize(1000)));
         assertThat(transaction.getSource(), is("Dividende14.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 59788848"));
@@ -3029,6 +3053,7 @@ public void testDividende15()
         assertThat(transaction.getType(), is(AccountTransaction.Type.TAXES));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2017-10-20T00:00")));
+        assertThat(transaction.getExDate(), is(nullValue()));
         assertThat(transaction.getShares(), is(Values.Share.factorize(0.4512)));
         assertThat(transaction.getSource(), is("Dividende15.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 12345678 | Ertrag für 2017"));
@@ -3073,6 +3098,7 @@ public void testDividende16()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2022-09-30T00:00")));
+        assertNull(transaction.getExDate());
         assertThat(transaction.getShares(), is(Values.Share.factorize(15)));
         assertThat(transaction.getSource(), is("Dividende16.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 42739637 | Kapitalrückzahlung"));
@@ -3117,6 +3143,7 @@ public void testDividende16WithSecurityInEUR()
         assertThat(transaction.getType(), is(AccountTransaction.Type.DIVIDENDS));
 
         assertThat(transaction.getDateTime(), is(LocalDateTime.parse("2022-09-30T00:00")));
+        assertNull(transaction.getExDate());
         assertThat(transaction.getShares(), is(Values.Share.factorize(15)));
         assertThat(transaction.getSource(), is("Dividende16.txt"));
         assertThat(transaction.getNote(), is("Abrechnungs-Nr. 42739637 | Kapitalrückzahlung"));
@@ -3160,6 +3187,7 @@ public void testDividende17()
         // check dividends transaction
         assertThat(results, hasItem(dividend( //
                         hasDate("2019-09-09"), hasShares(60), //
+                        hasExDate("2019-08-30"), //
                         hasSource("Dividende17.txt"), //
                         hasNote("Abrechnungs-Nr. 26128781"), //
                         hasAmount("EUR", 7.65), hasGrossValue("EUR", 15.73), //
@@ -3194,6 +3222,7 @@ public void testDividende17WithSecurityInEUR()
         // check dividends transaction
         assertThat(results, hasItem(dividend( //
                         hasDate("2019-09-09"), hasShares(60), //
+                        hasExDate("2019-08-30"), //
                         hasSource("Dividende17.txt"), hasNote("Abrechnungs-Nr. 26128781"), //
                         hasAmount("EUR", 7.65), hasGrossValue("EUR", 15.73), //
                         hasTaxes("EUR", (39.00 / 9.9148) + 3.93 + 0.22), hasFees("EUR", 0.00), //
@@ -3234,6 +3263,7 @@ public void testDividende18()
         // check dividends transaction
         assertThat(results, hasItem(dividend( //
                         hasDate("2015-12-17T00:00"), hasShares(156.729), //
+                        hasExDate("2015-12-15"), //
                         hasSource("Dividende18.txt"), //
                         hasNote("Abrechnungs-Nr. 70187215 | Ertrag für 2014/15"), //
                         hasAmount("EUR", 7.68), hasGrossValue("EUR", 11.84), //
@@ -3278,6 +3308,8 @@ public void testDividendeStorno01()
 
         assertThat(((Transaction) cancellation.getSubject()).getDateTime(),
                         is(LocalDateTime.parse("2020-05-15T00:00")));
+        assertThat(((AccountTransaction) cancellation.getSubject()).getExDate(),
+                        is(LocalDateTime.parse("2020-04-29T00:00")));
         assertThat(((Transaction) cancellation.getSubject()).getShares(), is(Values.Share.factorize(46)));
         assertThat(((Transaction) cancellation.getSubject()).getSource(), is("DividendeStorno01.txt"));
         assertThat(((Transaction) cancellation.getSubject()).getNote(), is(

name.abuchen.portfolio/src/name/abuchen/portfolio/datatransfer/pdf/OnvistaPDFExtractor.java

@@ -566,15 +566,17 @@ private void addDividendeTransaction()
                                         // STK 50,000 21.04.2016 21.04.2016 EUR 0,200000
                                         // @formatter:on
                                         section -> section //
-                                                        .attributes("name", "isin", "name1", "currency") //
+                                                        .attributes("exDate", "name", "isin", "name1", "currency") //
                                                         .find("Gattungsbezeichnung ISIN") //
                                                         .match("^(?<name>.*) (?<isin>[A-Z]{2}[A-Z0-9]{9}[0-9])$") //
                                                         .match("^([\\d]{2}\\.[\\d]{2}\\.[\\d]{2,4} )?(?<name1>.*)$") //
-                                                        .match("^STK [\\.,\\d]+ [\\d]{2}\\.[\\d]{2}\\.[\\d]{2,4} [\\d]{2}\\.[\\d]{2}\\.[\\d]{2,4} (?<currency>[A-Z]{3}) [\\.,\\d]+$") //
+                                                        .match("^STK [\\.,\\d]+ (?<exDate>[\\d]{2}\\.[\\d]{2}\\.[\\d]{2,4}) [\\d]{2}\\.[\\d]{2}\\.[\\d]{2,4} (?<currency>[A-Z]{3}) [\\.,\\d]+$") //
                                                         .assign((t, v) -> {
                                                             if (!v.get("name1").startsWith("Nominal"))
                                                                 v.put("name", trim(v.get("name")) + " " + trim(v.get("name1")));
 
+                                                            t.setExDate(asDate(v.get("exDate")));
+
                                                             t.setSecurity(getOrCreateSecurity(v));
                                                         }),
                                         // @formatter:off
@@ -914,6 +916,13 @@ private void addDividendeTransaction()
                             if (ctx.getString(FAILURE) != null)
                                 item.setFailureMessage(ctx.getString(FAILURE));
 
+                            // ex-date is only supported for dividend
+                            // transactions. if there is a
+                            // 'Ertragsthesaurierung', then we have a tax
+                            // transaction without ex-date.
+                            if (t.getType() != AccountTransaction.Type.DIVIDENDS && t.getExDate() != null)
+                                t.setExDate(null);
+
                             return item;
                         });