Squashed history before last 100 commits

Author: githubering182

.dockerignore

@@ -4,7 +4,6 @@
 db
 venv
 __pycache__
-migrations
 file_store
 storage
 worker_tmp

Makefile

@@ -1,41 +1,55 @@
-prod:
-	docker compose up -d
+include .env
 
-prod-new:
-	docker compose up -d --build
+DOCKER_IMAGE := dc_app
+DB_CONTAINER := iss-main-db
+PROD_FILE := -f docker-compose.yml
+DEV_FILE := -f docker-compose.dev.yml
+TEST_FILE := -f docker-compose.test.yml
 
-prod-stop:
-	docker compose down
+# MAIN
+build:
+	docker compose ${PROD_FILE} build --no-cache
 
-prod-restart:
-	make prod-stop && make prod
+start:
+	docker compose ${PROD_FILE} up -d
 
-dev:
-	docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d
+stop:
+	docker compose ${PROD_FILE} down
+
+start-new: build start
+restart: stop start
 
-dev-new:
-	docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
+# DEV
+dev:
+	docker compose ${PROD_FILE} ${DEV_FILE} up -d
 
 dev-stop:
-	docker compose -f docker-compose.yml -f docker-compose.dev.yml down
+	docker compose ${PROD_FILE} ${DEV_FILE} down
 
-dev-restart:
-	make dev-stop && make dev
+dev-new: build dev
+dev-restart: dev-stop dev
 
-test:
-	docker compose -f docker-compose.test.yml up -d --build
+# TEST
+test-start:
+	docker compose ${TEST_FILE} up -d
+
+test-build:
+	docker compose ${TEST_FILE} build --no-cache
 
 test-stop:
-	docker compose -f docker-compose.test.yml down
+	docker compose ${TEST_FILE} down
+
+test: test-build test-start
+test-restart: test-stop test
 
-test-restart:
-	make test-stop && make test
+# UTILS
+dump-schema:
+	docker exec ${DB_CONTAINER} pg_dump -U postgres -d ${DB_APP_DB_NAME} --schema-only > dump_schema
 
-appdb-dump-schema:
-	docker exec iss-main-db pg_dump -U postgres -d iss_app_db --schema-only > app_dump_schema
+dump-data:
+	docker exec ${DB_CONTAINER} pg_dump -U postgres -d ${DB_APP_DB_NAME} --data-only > dump_data
 
-appdb-dump-data:
-	docker exec iss-main-db pg_dump -U postgres -d iss_app_db --data-only > app_dump_data
+dump-all: dump-schema dump-data
 
-dump-database:
-	make appdb-dump-schema && make appdb-dump-data
+init-admin:
+	docker exec -it iss-back ./manage.py createsuperuser

README.md

@@ -1,94 +1,118 @@
+<img src="/docs/assets/preview.gif" alt="example">
+
 # ISS Data Collection Tool
 
-## Makefile convinient commands (**docker/docker-compose** and **.env file** are supposed to be set):
-
-- start project:
-  `make prod`
-- stop project:
-  `make prod-stop`
-- restart project:
-  `make prod-restart`
-- start project with rebuild:
-  `make prod-new`
-- start development project:
-  `make dev`
-- stop development project:
-  `make dev-stop`
-- restart development project:
-  `make dev-restart`
-- start project with rebuild:
-  `make dev-new`
-- start tests:
-  `make test`
-- stop tests:
-  `make test-stop`
-- restart tests:
-  `make test-restart`
-- dump main apps database schema:
-  `make appdb-dump-schema`
-- dump main apps database data:
-  `make appdb-dump-data`
-- dump main apps database (schema and data separately):
-  `make dump-database`
-
-## Prerequisites:
-
-- **docker/docker-compose** installed
-- create and fill **.env** file from sample
-
-For local development copying sample is enough:
-`cp .env.sample .env`
-
-## Running Application
-
-Docker Compose file: docker-compose.yml
-
-Docker files:
-
-- Dockerfile.backend
-- Dockerfile.frontend
-- Dockerfile.storage
-
-Command:
-`docker-compose up -d --build`
-
-## Development
-
-Docker Compose file: docker-compose.dev.yml
-
-Docker files:
-
-- Dockerfile.backend
-- Dockerfile.frontend
-- Dockerfile.storage
-
-Command:
-`docker-compose -f docker-compose.dev.yml up -d --build`
-
-## Testing
-
-Docker Compose file: docker-compose.test.yml
-
-Docker files:
-
-- Dockerfile.tests
-
-Command (rebuild is important):
-`docker-compose -f docker-compose.test.yml up -d --build`
-
-Available tests:
-
-- Main Backend:
-  `docker exec iss-test-back ./manage.py test`
-- Storage Backend:
-  `docker exec iss-test-storage python3 src/test.py`
-- Frontend:
-  `docker exec iss-test-front npm test`
-- Selenium Tests (browser emulation):
-  `docker exec iss-tests python3 test.py`
-- Python linter (no output means the lint test is passed):
-  `docker exec iss-tests flake8`
-- JavaScript linter:
-  `docker exec iss-test-front npm run lint`
-- JavaScript ts compiler checker:
-  `docker exec iss-test-front npm run compile`
+An end-to-end dataset collection system designed for scalability. Supports multi-role workflows, structured label taxonomies, validation cycles, goal tracking, and archive exports. Ideal for organizations building private, high-integrity datasets with distributed teams of data collectors.
+
+🛠 Currently in active development. Ideal for internal use, pilots, and research-stage projects.
+
+## 🧩 Features
+
+This platform enables you to:
+
+- Create projects with custom label systems
+- Upload images/videos and assign them to labeling schemas
+- validate uploaded files
+- Set collection goals
+- Track progress with stats
+- Export data
+
+## 📚 Documentation & Examples
+
+See [docs/](/docs) for manuals and walkthroughs:
+
+- [Quickstart](/docs/quickstart.md)
+- [Projects](/docs/projects.md)
+- [Labels](/docs/labels.md)
+- [Users and Roles](/docs/users.md)
+- [Uploads](/docs/uploads.md)
+- [Validation](/docs/validation.md)
+- [Goals](/docs/goals.md)
+- [Statistics](/docs/statistics.md)
+- [Downloads](/docs/downloads.md)
+- [Annotation](/docs/annotation.md)
+
+## ⚙️ Architecture
+
+- **Main Backend:** Django + PostgreSQL
+- **File Backend:** FastAPI + MongoDB (blob storage)
+- **Task Queue:** Celery + Redis
+- **Frontend:** React
+- **Deployment:** Docker, Compose, Makefile-based workflow
+
+## 📁 Folder Structure (Top Level)
+
+- `backend-app/` — main Django app
+- `frontend-app/` — React app
+- `storage-app/` — FastAPI blob service
+- `scripts/` — app handy tools
+- `tests/` — global tests
+- `nginx/`, `redis/` — infrastructure configs
+- `Makefile` — common commands
+- `docker-compose*.yml` — dev/test/prod setup
+
+## 🚀 Getting Started
+
+### Prerequisites
+
+- Docker + Docker Compose installed
+- `.env` file created from `.env.sample`
+
+```bash
+cp .env.sample .env
+```
+
+### Build & Run
+```bash
+make build  # build all services
+make start  # start in prod mode
+make dev    # start in dev mode
+```
+Full command list available in the Makefile section below.
+
+## 🧪 Testing
+Run with:
+```bash
+docker exec iss-test-back ./manage.py test          # Main Backend
+docker exec iss-test-storage python3 src/test.py    # Storage Backend
+docker exec iss-test-front npm test                 # Frontend
+docker exec iss-tests flake8                        # Python linter
+docker exec iss-test-front npm run lint             # JavaScript linter
+docker exec iss-test-front npm run compile          # JavaScript ts compiler checker
+```
+
+## 🛠️ Makefile Commands
+
+General
+```bash
+make build          # build all services
+make start          # start in prod mode
+make stop           # stop prod mode
+make start-new      # rebuild and start services in prod mode
+make restart        # stop and start in prod mode
+```
+
+Dev Mode
+```bash
+make dev            # start in dev mode
+make dev-stop       # stop dev mode
+make dev-new        # rebuild and start services in dev mode
+make dev-restart    # stop and start in dev mode
+```
+
+Tests
+```bash
+make test           # rebuild and start services in test mode
+make test-start     # start in test mode
+make test-build     # build test mode
+make test-stop      # stop test mode
+make test-restart   # stop and start services in test mode
+```
+
+Utils
+```bash
+make dump-schema    # dump database schema
+make dump-data      # dump database data
+make dump-all       # dump database both schema and data
+make init-admin     # create new superuser
+```

backend-app/archive/__init__.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ArchiveConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'archive'

backend-app/archive/apps.py

@@ -0,0 +1,30 @@
+from django.db.models import (
+    Model,
+    CharField,
+    ForeignKey,
+    ManyToManyField,
+    DO_NOTHING,
+    DateTimeField,
+    BigIntegerField,
+    JSONField,
+    TextField
+)
+
+
+class Archive(Model):
+    STATUSES = (("s", "SUCCESS"), ("f", "FAILURE"), ("p", "PENDING"),)
+
+    id = CharField(max_length=36, primary_key=True)
+    result_id = CharField(max_length=24, null=True)
+    result_size = BigIntegerField(null=True)
+    result_message = TextField(null=True)
+    status = CharField(max_length=1, choices=STATUSES, default="p")
+    filters = JSONField()
+    create_date = DateTimeField(auto_now_add=True)
+
+    project = ForeignKey("project.Project", on_delete=DO_NOTHING)
+    author = ForeignKey("user.CustomUser", on_delete=DO_NOTHING)
+    file = ManyToManyField("file.File")
+
+    class Meta:
+        db_table = "archive"

backend-app/archive/models.py

@@ -0,0 +1,15 @@
+from rest_framework.permissions import BasePermission
+from rest_framework.views import Request, APIView
+
+
+class ArchivesPermission(BasePermission):
+    def has_permission(self, request: Request, view: APIView) -> bool | None:
+        project_perm = bool(request.user.project_download.filter(id=view.kwargs["p_pk"]))
+        return request.user.is_superuser or project_perm
+
+
+class ArchivePermission(BasePermission):
+    def has_permission(self, request: Request, view: APIView) -> bool | None:
+        project_perm = bool(request.user.project_download.filter(id=view.kwargs["p_pk"]))
+        arch_perm = bool(request.user.archive_set.filter(id=view.kwargs["pk"]))
+        return request.user.is_superuser or (project_perm and arch_perm)

backend-app/archive/permissions.py

@@ -0,0 +1,18 @@
+from rest_framework.serializers import ModelSerializer, ChoiceField, SerializerMethodField, IntegerField
+from .models import Archive
+
+
+class StatusChoiceField(ChoiceField):
+    def to_representation(self, value: str) -> str: return self.choices[value]
+
+
+class ArchiveSerializer(ModelSerializer):
+    status = StatusChoiceField(choices=Archive.STATUSES)
+    author = SerializerMethodField()
+    file_count = IntegerField()
+
+    class Meta:
+        model = Archive
+        exclude = ("file", "project")
+
+    def get_author(self, instance: Archive): return instance.author.username