Merge branch 'master' into feature/excel_import

Author: Hironsan

.travis.yml

@@ -6,6 +6,7 @@ python:
 env:
   - DATABASE=sqlite
   - DATABASE=postgres
+  - DATABASE=mssql
 
 services:
   - docker
@@ -16,6 +17,11 @@ before_install:
     if [[ "${DATABASE}" = "postgres" ]]; then
       docker run --rm --name=postgres --network=doccano -d -e POSTGRES_USER=user -e POSTGRES_PASSWORD=pass -e POSTGRES_DB=db postgres
       export DATABASE_URL="postgres://user:pass@postgres:5432/db?sslmode=disable"
+
+    elif [[ "${DATABASE}" = "mssql" ]]; then
+      docker run --rm --name=mssql --network=doccano -d -e ACCEPT_EULA=y -e SA_PASSWORD=sUp3rS3cr3t mcr.microsoft.com/mssql/server:2017-latest
+      docker exec -it mssql sh -c "while ! /opt/mssql-tools/bin/sqlcmd -U SA -P sUp3rS3cr3t -Q 'CREATE DATABASE db;'; do sleep 3; done"
+      export DATABASE_URL="mssql://SA:sUp3rS3cr3t@mssql:1433/db?sslmode=disable"
     fi
 
 install:
@@ -25,7 +31,7 @@ script:
   - docker build --target=builder --tag=doccano-test .
   - >
     if [[ "${DATABASE}" != "sqlite" ]]; then
-      docker run --network doccano -e DATABASE_URL="${DATABASE_URL}" -it doccano-test sh -c 'app/manage.py migrate && app/manage.py test api.tests server.tests'
+      docker run --network doccano -e DATABASE_URL="${DATABASE_URL}" -it doccano-test sh -c 'app/manage.py wait_for_db && app/manage.py migrate && app/manage.py test api.tests server.tests'
     fi
 
 before_deploy:

Dockerfile

@@ -4,10 +4,10 @@ FROM python:${PYTHON_VERSION}-stretch AS builder
 ARG NODE_VERSION="8.x"
 RUN curl -sL "https://deb.nodesource.com/setup_${NODE_VERSION}" | bash - \
  && apt-get install --no-install-recommends -y \
-      nodejs=8.16.0-1nodesource1
+      nodejs
 
-RUN apt-get install --no-install-recommends -y \
-      unixodbc-dev=2.3.4-1
+COPY tools/install-mssql.sh /doccano/tools/install-mssql.sh
+RUN /doccano/tools/install-mssql.sh --dev
 
 COPY app/server/static/package*.json /doccano/app/server/static/
 RUN cd /doccano/app/server/static \
@@ -33,19 +33,8 @@ RUN cd /doccano \
 
 FROM python:${PYTHON_VERSION}-slim-stretch AS runtime
 
-RUN apt-get update \
- && apt-get install --no-install-recommends -y \
-      curl=7.52.1-5+deb9u9 \
-      gnupg=2.1.18-8~deb9u4 \
-      apt-transport-https=1.4.9 \
- && curl -fsS https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \
- && curl -fsS https://packages.microsoft.com/config/debian/9/prod.list > /etc/apt/sources.list.d/mssql.list \
- && apt-get update \
- && ACCEPT_EULA=Y apt-get install --no-install-recommends -y \
-      msodbcsql17=17.3.1.1-1 \
-      mssql-tools=17.3.0.1-1 \
- && apt-get remove -y curl gnupg apt-transport-https \
- && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /doccano/tools/install-mssql.sh /doccano/tools/install-mssql.sh
+RUN /doccano/tools/install-mssql.sh
 
 RUN useradd -ms /bin/sh doccano
 

Procfile

@@ -124,21 +124,19 @@ Let’s start the development server and explore it.
 
 Depending on your installation method, there are two options:
 
-**Option1: Running the Docker image as a Container**
+#### Option 1: Running the Docker image as a Container
 
 First, run a Docker container:
 
 ```bash
-docker run -d --name doccano -p 8000:8000 chakkiworks/doccano
+docker run -d --rm --name doccano \
+  -e "ADMIN_USERNAME=admin" \
+  -e "[email protected]" \
+  -e "ADMIN_PASSWORD=password" \
+  -p 8000:8000 chakkiworks/doccano
 ```
 
-Then, execute `create-admin.sh` script for creating a superuser.
-
-```bash
-docker exec doccano tools/create-admin.sh "admin" "[email protected]" "password"
-```
-
-**Option2: Running Django development server**
+#### Option 2: Running Django development server
 
 Before running, we need to make migration. Run the following command:
 
@@ -170,7 +168,7 @@ Optionally, you can change the bind ip and port using the command
 python manage.py runserver <ip>:<port>
 ```
 
-**Option3: Running the development Docker-Compose stack**
+#### Option 3: Running the development Docker-Compose stack
 
 We can use docker-compose to set up the webpack server, django server, database, etc. all in one command:
 
@@ -259,14 +257,14 @@ Input file may look like this:
 `import.json`
 
 ```JSON
-{"text": "EU rejects German call to boycott British lamb.", "external_id": 1}
+{"text": "EU rejects German call to boycott British lamb.", "meta": {"external_id": 1}}
 ```
 
 and the exported file will look like this:
 `output.json`
 
 ```JSON
-{"doc_id": 2023, "text": "EU rejects German call to boycott British lamb.", "labels": ["news"], "username": "root", "metadata": {"external_id": 1}}
+{"doc_id": 2023, "text": "EU rejects German call to boycott British lamb.", "labels": ["news"], "username": "root", "meta": {"external_id": 1}}
 ```
 
 ### Tutorial

README.md

@@ -36,20 +36,13 @@
             "value": "True"
         }
     },
+    "stack": "container",
     "scripts": {
-        "postdeploy": "sh tools/heroku.sh deploy"
+        "postdeploy": "sh tools/heroku.sh"
     },
     "addons": [
         {
             "plan": "heroku-postgresql:hobby-dev"
         }
-    ],
-    "buildpacks": [
-        {
-            "url": "heroku/nodejs"
-        },
-        {
-            "url": "heroku/python"
-        }
     ]
 }

app.json

@@ -0,0 +1,33 @@
+from collections import Counter
+
+from django.db.models import Manager, Count
+
+
+class AnnotationManager(Manager):
+
+    def get_label_per_data(self, project):
+        label_count = Counter()
+        user_count = Counter()
+        docs = project.documents.all()
+        annotations = self.filter(document_id__in=docs.all())
+
+        for d in annotations.values('label__text', 'user__username').annotate(Count('label'), Count('user')):
+            label_count[d['label__text']] += d['label__count']
+            user_count[d['user__username']] += d['user__count']
+
+        return label_count, user_count
+
+
+class Seq2seqAnnotationManager(Manager):
+
+    def get_label_per_data(self, project):
+        label_count = Counter()
+        user_count = Counter()
+        docs = project.documents.all()
+        annotations = self.filter(document_id__in=docs.all())
+
+        for d in annotations.values('text', 'user__username').annotate(Count('text'), Count('user')):
+            label_count[d['text']] += d['text__count']
+            user_count[d['user__username']] += d['user__count']
+
+        return label_count, user_count

app/api/managers.py

@@ -7,6 +7,8 @@
 from django.core.exceptions import ValidationError
 from polymorphic.models import PolymorphicModel
 
+from .managers import AnnotationManager, Seq2seqAnnotationManager
+
 DOCUMENT_CLASSIFICATION = 'DocumentClassification'
 SEQUENCE_LABELING = 'SequenceLabeling'
 SEQ2SEQ = 'Seq2seq'
@@ -191,6 +193,8 @@ def __str__(self):
 
 
 class Annotation(models.Model):
+    objects = AnnotationManager()
+
     prob = models.FloatField(default=0.0)
     manual = models.BooleanField(default=False)
     user = models.ForeignKey(User, on_delete=models.CASCADE)
@@ -224,6 +228,9 @@ class Meta:
 
 
 class Seq2seqAnnotation(Annotation):
+    # Override AnnotationManager for custom functionality
+    objects = Seq2seqAnnotationManager()
+
     document = models.ForeignKey(Document, related_name='seq2seq_annotations', on_delete=models.CASCADE)
     text = models.CharField(max_length=500)
 

app/api/models.py

@@ -160,5 +160,5 @@ class Seq2seqAnnotationSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Seq2seqAnnotation
-        fields = ('id', 'text', 'user', 'document')
+        fields = ('id', 'text', 'user', 'document', 'prob')
         read_only_fields = ('user',)

app/api/serializers.py

@@ -1,4 +1,4 @@
-SOCCERO
+SOCCERO SOCCERO SOCCERO
 -	O
 JAPAN	B-LOC
 GET	O

app/api/tests/data/labeling.invalid.conll

@@ -143,13 +143,14 @@ def test_make_annotations(self):
 
 class TestCoNLLParser(TestCase):
     def test_calc_char_offset(self):
-        words = ['EU', 'rejects', 'German', 'call']
-        tags = ['B-ORG', 'O', 'B-MISC', 'O']
-
-        entities = get_entities(tags)
-        actual = CoNLLParser.calc_char_offset(words, tags)
-
-        self.assertEqual(entities, [('ORG', 0, 0), ('MISC', 2, 2)])
+        f = io.BytesIO(
+          b"EU\tORG\n"
+          b"rejects\t_\n"
+          b"German\tMISC\n"
+          b"call\t_\n"
+        )
+
+        actual = next(CoNLLParser().parse(f))[0]
 
         self.assertEqual(actual, {
             'text': 'EU rejects German call',