Merge pull request #200 from rambo/ndaparser_metadata_frommaster

Keep track of uploaded Nordea transaction files and store last upload time

Author: rambo

project/ndaparser/migrations/0001_initial.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import asylum.mixins
+import ndaparser.models
+from django.conf import settings
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='UploadedTransaction',
+            fields=[
+                ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)),
+                ('file', models.FileField(upload_to=ndaparser.models.datestamped_and_normalized)),
+                ('stamp', models.DateTimeField(auto_now_add=True)),
+                ('last_transaction', models.DateField()),
+                ('user', models.ForeignKey(on_delete=models.SET(ndaparser.models.get_sentinel_user), to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'verbose_name': 'Uploaded transaction',
+                'verbose_name_plural': 'Uploaded transaction',
+                'ordering': ['-stamp'],
+            },
+            bases=(asylum.mixins.AtomicVersionMixin, asylum.mixins.CleanSaveMixin, models.Model),
+        ),
+    ]

project/ndaparser/models.py

@@ -1,6 +1,48 @@
 # -*- coding: utf-8 -*-
+import datetime
+import slugify as unicodeslugify
+
 from django.db import models, transaction
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.utils.translation import ugettext_lazy as _
+
 
 from asylum.models import AsylumModel
 
-# Create your models here.
+
+
+def get_sentinel_user():
+    """Gets a "sentinel" user ("deleted") and for assigning as uploader"""
+    return get_user_model().objects.get_or_create(username='deleted')[0]
+
+
+
+def datestamped_and_normalized(instance, filename):
+    """Normalized filename and places in datestamped path"""
+    file_parts = filename.split('.')
+    if len(file_parts) > 1:
+        name = '.'.join(file_parts[:-1])
+        ext = '.' + file_parts[-1]
+    else:
+        ext = ''
+        name = filename
+    filename_normalized = unicodeslugify.slugify(
+        name, only_ascii=True, lower=True,
+        spaces=False, space_replacement='_'
+    ) + ext
+    return datetime.datetime.now().strftime("ndaparser/%Y/%m/%d/{}").format(filename_normalized)
+
+
+
+class UploadedTransaction(AsylumModel):
+    """Track uploaded transaction files"""
+    user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET(get_sentinel_user))
+    file = models.FileField(upload_to=datestamped_and_normalized)
+    stamp = models.DateTimeField(auto_now_add=True, editable=False)
+    last_transaction = models.DateField()
+
+    class Meta:
+        verbose_name = _('Uploaded transaction')
+        verbose_name_plural = _('Uploaded transaction')
+        ordering = [ '-stamp' ]

project/ndaparser/parser.py

@@ -128,7 +128,7 @@ def ascii2scandic(string):
 
 if __name__ == "__main__":
     transactions = []
-    with open("./testdata.nda") as f:
+    with open("./tests/testdata.nda") as f:
         for line in f:
             transaction = parseLine(line)
             if transaction is not None:

project/ndaparser/tests.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

project/ndaparser/tests/__init__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+import os
+
+import pytest
+from ndaparser.parser import parseLine
+
+
+def test_parser():
+    transactions = []
+    with open(os.path.join(os.path.dirname(__file__), "testdata.nda")) as f:
+        for line in f:
+            transaction = parseLine(line)
+            if transaction is not None:
+                transactions.append(transaction)
+    assert len(transactions) == 12
+    assert transactions[2].amount == 40

project/ndaparser/tests/test_parser.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+import datetime
+import os
+
+import pytest
+from django.core.files.uploadedfile import SimpleUploadedFile
+from ndaparser.models import UploadedTransaction
+
+
+@pytest.mark.django_db
+def test_model_can_be_saved(admin_user):
+    with open(os.path.join(os.path.dirname(__file__), "testdata.nda"), 'rb') as f:
+        ut = UploadedTransaction(
+            last_transaction=datetime.datetime.now().date(),
+            user=admin_user,
+            file=SimpleUploadedFile('pytest.nda', f.read())
+        )
+    ut.save()
+    assert ut.pk > 0

project/ndaparser/tests/test_saving.py

@@ -7,6 +7,8 @@
 
 from .forms import UploadForm
 from .importer import NDAImporter
+from .parser import parseLine
+from .models import UploadedTransaction
 
 
 class NordeaUploadView(FormView):
@@ -32,6 +34,24 @@ def form_valid(self, form):
             h = NDAImporter(f)
             transactions = h.import_transactions(transactions_handler)
 
+        # parse the file again for the last transaction timestamp
+        last_stamp = None
+        with open(tmp.name) as fp:
+            for line in fp:
+                nt =  parseLine(line)
+                if not nt:
+                    continue
+                if not last_stamp:
+                    last_stamp = nt.timestamp
+                if nt.timestamp > last_stamp:
+                    last_stamp = nt.timestamp
+
+        UploadedTransaction(
+            last_transaction = last_stamp,
+            file = self.request.FILES['ndafile'],
+            user = self.request.user
+        ).save()
+
         # Done with the temp file, get rid of it
         os.unlink(tmp.name)