diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..ba22655 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,85 @@ +name: Build + +on: + push: null + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + architecture: x64 + - name: Set Gradle User Home + run: export GRADLE_USER_HOME=$(pwd)/.gradle + - name: Cache Gradle Dependencies + uses: actions/cache@v2 + with: + path: .gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: ${{ runner.os }}-gradle + - name: Cache Gradle Wrapper + uses: actions/cache@v2 + with: + path: .gradle/wrapper + key: ${{ runner.os }}-gradle-wrapper-${{ hashFiles('**/gradle-wrapper.properties') }} + restore-keys: ${{ runner.os }}-gradle + - name: Gradle build + run: ./gradlew clean build --parallel + prepareDockerImage: + needs: build + name: Prepare docker images + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + architecture: x64 + - name: Build Jar + run: ./gradlew bootJar + - name: Build login + run: docker login -u ${{ vars.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }} + - name: Build docker image + run: docker build -t iceknight07/open-chat:latest . + - name: Push docker image + run: docker push iceknight07/open-chat:latest + deploy: + needs: prepareDockerImage + name: Deploy + runs-on: ubuntu-latest + steps: + - name: Setup SSH connection + run: | + eval $(ssh-agent -s) + mkdir -p ~/.ssh + chmod 700 ~/.ssh + ssh-keyscan ${{ vars.DEPLOY_HOST }} >> ~/.ssh/known_hosts + chmod 644 ~/.ssh/known_hosts + echo "${{ secrets.DEPLOY_KEY }}" | tr -d '\r' > ~/.ssh/private.key + chmod 600 ~/.ssh/private.key + - name: Run command + run: | + ssh -i ~/.ssh/private.key ${{ vars.DEPLOY_USER }}@${{ vars.DEPLOY_HOST }} \ + "docker stop open-chat-server || true" + ssh -i ~/.ssh/private.key ${{ vars.DEPLOY_USER }}@${{ vars.DEPLOY_HOST }} \ + "docker rm open-chat-server || true" + ssh -i ~/.ssh/private.key ${{ vars.DEPLOY_USER }}@${{ vars.DEPLOY_HOST }} \ + "docker rmi iceknight07/open-chat:latest || true" + ssh -i ~/.ssh/private.key ${{ vars.DEPLOY_USER }}@${{ vars.DEPLOY_HOST }} \ + "docker run --name open-chat-server --network=open-chat-network -p 443:8443 -d \ + -e POSTGRES_URL=open-chat-postgres:5432/open_chat \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=12345678 \ + -e KMS_URL=ws://kurento-media-server:8888/kurento \ + -e SPRING_PROFILES_ACTIVE=production \ + iceknight07/open-chat:latest" \ No newline at end of file diff --git a/build-docker-image.sh b/build-docker-image.sh new file mode 100755 index 0000000..3acb858 --- /dev/null +++ b/build-docker-image.sh @@ -0,0 +1,3 @@ +VERSION=5.4 +docker build --platform linux/amd64 -t iceknight07/open-chat:$VERSION . +docker push iceknight07/open-chat:$VERSION \ No newline at end of file diff --git a/build.gradle b/build.gradle index dab9f89..efc2af0 100644 --- a/build.gradle +++ b/build.gradle @@ -21,17 +21,23 @@ configurations { repositories { mavenCentral() + maven { + url "https://maven.dcm4che.org" + } } dependencies { implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' implementation 'org.jetbrains.kotlin:kotlin-reflect' + implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.0" + implementation 'io.jsonwebtoken:jjwt:0.12.5' implementation 'io.jsonwebtoken:jjwt-api:0.12.5' implementation 'javax.xml.bind:jaxb-api:2.1' implementation 'com.fasterxml.jackson.module:jackson-module-kotlin:2.17.1' + implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.17.1' runtimeOnly("com.mysql:mysql-connector-j") @@ -39,7 +45,9 @@ dependencies { implementation 'org.flywaydb:flyway-core:10.10.0' implementation "org.flywaydb:flyway-database-postgresql:10.10.0" + implementation 'org.springframework.data:spring-data-relational:3.2.3' implementation "org.springframework.boot:spring-boot-starter-data-jpa" + implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.boot:spring-boot-starter-security' implementation 'org.springframework.boot:spring-boot-starter-web' implementation("org.springframework.boot:spring-boot-starter-validation") @@ -49,16 +57,42 @@ dependencies { implementation("com.amazonaws:aws-java-sdk-cognitoidp:1.12.681") implementation 'com.amazonaws:aws-java-sdk-s3:1.12.687' + implementation 'com.amazonaws:aws-java-sdk-transcribe:1.12.761' + implementation 'software.amazon.awssdk:transcribestreaming:2.26.20' + implementation("javax.validation:validation-api:2.0.1.Final") implementation 'com.amazonaws:aws-java-sdk-qconnect:1.12.693' + implementation 'xuggle:xuggle-xuggler:5.4' implementation("org.passay:passay:1.6.3") + implementation 'org.kurento:kurento-client:7.1.0' + + // WebJars + // WebJars Locator + implementation 'org.webjars:webjars-locator:0.46' + + // + implementation ("org.webjars.bower:bootstrap:5.2.2") + implementation ("org.webjars.bower:demo-console:1.5.1") + implementation ("org.webjars.bower:draggabilly:2.1.0") + implementation ("org.webjars.bower:ekko-lightbox:5.2.0") + implementation ("org.webjars.bower:jquery:3.6.1") + implementation ("org.webjars.bower:jsnlog.js:2.20.1") + implementation ("org.webjars.bower:webrtc-adapter:7.4.0") + + implementation("org.kurento:kurento-commons:7.1.0") + implementation("org.kurento:kurento-client:7.1.0") + implementation("org.kurento:kurento-utils-js:7.1.0") + compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' annotationProcessor 'org.projectlombok:lombok' testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.springframework.security:spring-security-test' + + testImplementation 'org.testcontainers:postgresql:1.19.7' + } tasks.named('test') { diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml index 186bac9..4b441d8 100644 --- a/docker-compose/docker-compose.yaml +++ b/docker-compose/docker-compose.yaml @@ -27,6 +27,7 @@ services: open-chat-postgres: container_name: open-chat-postgres + hostname: open-chat-postgres image: postgres environment: POSTGRES_DB: open_chat diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 6e59778..52e6214 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -15,7 +15,7 @@ # distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/meeting-recording.json b/meeting-recording.json new file mode 100644 index 0000000..6e12d2b --- /dev/null +++ b/meeting-recording.json @@ -0,0 +1,1912 @@ +{ + "jobName": "TranscriptionJob1-2024-12-09_21-06-00-11-combined.webm-d12b3469-0646-4dff-bbd6-b67913e2da13", + "accountId": "533267044720", + "status": "COMPLETED", + "results": { + "transcripts": [ + { + "transcript": "The currency Cryptocurrency or crypto is a digital currency designed to work through a computer network that is not the one of any central authority such as government or bank job hold or maintain it individual coin own records. A to digital ledger which is computerized database, using strong Cryptocurrency to secure transaction codes control the creation of additional coins to verify the transfer co ownership. Despite the term that has come to describe many of the funds of production tokens have been created. Cryptocurrency are not considered currency, traditional things and very tr treatments have implied to the area. Various jurisdictions include the classification of commodities, securities and currencies. Cryptocurrency are generally viewed as this cost cost. In practice, some crypto schemes use the data to maintain the Cryptocurrency. Thank you." + } + ], + "items": [ + { + "id": 0, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.58", + "content": "The" + } + ], + "start_time": "4.219", + "end_time": "4.239" + }, + { + "id": 1, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.343", + "content": "currency" + } + ], + "start_time": "4.679", + "end_time": "4.86" + }, + { + "id": 2, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.982", + "content": "Cryptocurrency" + } + ], + "start_time": "5.039", + "end_time": "5.909" + }, + { + "id": 3, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "or" + } + ], + "start_time": "5.929", + "end_time": "6.46" + }, + { + "id": 4, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.485", + "content": "crypto" + } + ], + "start_time": "6.469", + "end_time": "6.82" + }, + { + "id": 5, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.961", + "content": "is" + } + ], + "start_time": "7.25", + "end_time": "7.46" + }, + { + "id": 6, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.827", + "content": "a" + } + ], + "start_time": "7.63", + "end_time": "7.639" + }, + { + "id": 7, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.805", + "content": "digital" + } + ], + "start_time": "7.789", + "end_time": "7.989" + }, + { + "id": 8, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.993", + "content": "currency" + } + ], + "start_time": "8.0", + "end_time": "8.479" + }, + { + "id": 9, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.993", + "content": "designed" + } + ], + "start_time": "8.489", + "end_time": "9.01" + }, + { + "id": 10, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.991", + "content": "to" + } + ], + "start_time": "9.02", + "end_time": "9.109" + }, + { + "id": 11, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "work" + } + ], + "start_time": "9.119", + "end_time": "9.55" + }, + { + "id": 12, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.81", + "content": "through" + } + ], + "start_time": "9.56", + "end_time": "9.659" + }, + { + "id": 13, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.684", + "content": "a" + } + ], + "start_time": "9.689", + "end_time": "9.699" + }, + { + "id": 14, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.912", + "content": "computer" + } + ], + "start_time": "9.71", + "end_time": "9.859" + }, + { + "id": 15, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.997", + "content": "network" + } + ], + "start_time": "10.159", + "end_time": "10.8" + }, + { + "id": 16, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.902", + "content": "that" + } + ], + "start_time": "11.17", + "end_time": "11.359" + }, + { + "id": 17, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "is" + } + ], + "start_time": "11.369", + "end_time": "11.52" + }, + { + "id": 18, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.989", + "content": "not" + } + ], + "start_time": "11.529", + "end_time": "11.579" + }, + { + "id": 19, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.881", + "content": "the" + } + ], + "start_time": "11.59", + "end_time": "11.609" + }, + { + "id": 20, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.783", + "content": "one" + } + ], + "start_time": "11.689", + "end_time": "12.13" + }, + { + "id": 21, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.934", + "content": "of" + } + ], + "start_time": "12.14", + "end_time": "12.279" + }, + { + "id": 22, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.988", + "content": "any" + } + ], + "start_time": "12.42", + "end_time": "12.59" + }, + { + "id": 23, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.981", + "content": "central" + } + ], + "start_time": "12.6", + "end_time": "13.02" + }, + { + "id": 24, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.907", + "content": "authority" + } + ], + "start_time": "13.039", + "end_time": "13.829" + }, + { + "id": 25, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.998", + "content": "such" + } + ], + "start_time": "14.02", + "end_time": "14.39" + }, + { + "id": 26, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "as" + } + ], + "start_time": "14.399", + "end_time": "14.449" + }, + { + "id": 27, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.968", + "content": "government" + } + ], + "start_time": "14.46", + "end_time": "15.029" + }, + { + "id": 28, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.923", + "content": "or" + } + ], + "start_time": "15.039", + "end_time": "15.06" + }, + { + "id": 29, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "bank" + } + ], + "start_time": "15.09", + "end_time": "15.51" + }, + { + "id": 30, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.748", + "content": "job" + } + ], + "start_time": "15.52", + "end_time": "15.739" + }, + { + "id": 31, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.649", + "content": "hold" + } + ], + "start_time": "15.979", + "end_time": "16.61" + }, + { + "id": 32, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.997", + "content": "or" + } + ], + "start_time": "16.62", + "end_time": "16.909" + }, + { + "id": 33, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "maintain" + } + ], + "start_time": "16.92", + "end_time": "17.45" + }, + { + "id": 34, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.978", + "content": "it" + } + ], + "start_time": "17.459", + "end_time": "17.649" + }, + { + "id": 35, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.855", + "content": "individual" + } + ], + "start_time": "18.53", + "end_time": "18.69" + }, + { + "id": 36, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.923", + "content": "coin" + } + ], + "start_time": "18.94", + "end_time": "19.44" + }, + { + "id": 37, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.599", + "content": "own" + } + ], + "start_time": "19.45", + "end_time": "19.729" + }, + { + "id": 38, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.994", + "content": "records" + } + ], + "start_time": "20.569", + "end_time": "21.25" + }, + { + "id": 39, + "type": "punctuation", + "alternatives": [ + { + "confidence": "0.0", + "content": "." + } + ] + }, + { + "id": 40, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.79", + "content": "A" + } + ], + "start_time": "21.26", + "end_time": "21.27" + }, + { + "id": 41, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.627", + "content": "to" + } + ], + "start_time": "21.29", + "end_time": "21.459" + }, + { + "id": 42, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "digital" + } + ], + "start_time": "21.77", + "end_time": "22.239" + }, + { + "id": 43, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.989", + "content": "ledger" + } + ], + "start_time": "22.25", + "end_time": "22.84" + }, + { + "id": 44, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.999", + "content": "which" + } + ], + "start_time": "22.85", + "end_time": "23.159" + }, + { + "id": 45, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "is" + } + ], + "start_time": "23.17", + "end_time": "23.25" + }, + { + "id": 46, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.769", + "content": "computerized" + } + ], + "start_time": "23.26", + "end_time": "23.899" + }, + { + "id": 47, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.518", + "content": "database" + } + ], + "start_time": "23.909", + "end_time": "24.53" + }, + { + "id": 48, + "type": "punctuation", + "alternatives": [ + { + "confidence": "0.0", + "content": "," + } + ] + }, + { + "id": 49, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.997", + "content": "using" + } + ], + "start_time": "24.54", + "end_time": "24.94" + }, + { + "id": 50, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "strong" + } + ], + "start_time": "24.95", + "end_time": "25.229" + }, + { + "id": 51, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.943", + "content": "Cryptocurrency" + } + ], + "start_time": "25.239", + "end_time": "26.29" + }, + { + "id": 52, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "to" + } + ], + "start_time": "26.85", + "end_time": "27.11" + }, + { + "id": 53, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "secure" + } + ], + "start_time": "27.12", + "end_time": "27.6" + }, + { + "id": 54, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.528", + "content": "transaction" + } + ], + "start_time": "27.68", + "end_time": "27.959" + }, + { + "id": 55, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.837", + "content": "codes" + } + ], + "start_time": "28.069", + "end_time": "28.43" + }, + { + "id": 56, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.999", + "content": "control" + } + ], + "start_time": "28.44", + "end_time": "28.829" + }, + { + "id": 57, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.998", + "content": "the" + } + ], + "start_time": "28.84", + "end_time": "28.93" + }, + { + "id": 58, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "creation" + } + ], + "start_time": "28.94", + "end_time": "29.34" + }, + { + "id": 59, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "of" + } + ], + "start_time": "29.35", + "end_time": "29.389" + }, + { + "id": 60, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.988", + "content": "additional" + } + ], + "start_time": "29.399", + "end_time": "29.78" + }, + { + "id": 61, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "coins" + } + ], + "start_time": "29.79", + "end_time": "30.219" + }, + { + "id": 62, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.992", + "content": "to" + } + ], + "start_time": "30.229", + "end_time": "30.28" + }, + { + "id": 63, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.989", + "content": "verify" + } + ], + "start_time": "30.29", + "end_time": "30.639" + }, + { + "id": 64, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "the" + } + ], + "start_time": "30.649", + "end_time": "30.77" + }, + { + "id": 65, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.992", + "content": "transfer" + } + ], + "start_time": "30.78", + "end_time": "31.159" + }, + { + "id": 66, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.969", + "content": "co" + } + ], + "start_time": "31.17", + "end_time": "31.389" + }, + { + "id": 67, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "ownership" + } + ], + "start_time": "31.399", + "end_time": "32.0" + }, + { + "id": 68, + "type": "punctuation", + "alternatives": [ + { + "confidence": "0.0", + "content": "." + } + ] + }, + { + "id": 69, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "Despite" + } + ], + "start_time": "32.659", + "end_time": "33.119" + }, + { + "id": 70, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.993", + "content": "the" + } + ], + "start_time": "33.13", + "end_time": "33.209" + }, + { + "id": 71, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.98", + "content": "term" + } + ], + "start_time": "33.22", + "end_time": "33.63" + }, + { + "id": 72, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.973", + "content": "that" + } + ], + "start_time": "33.639", + "end_time": "33.779" + }, + { + "id": 73, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.846", + "content": "has" + } + ], + "start_time": "33.79", + "end_time": "33.919" + }, + { + "id": 74, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.987", + "content": "come" + } + ], + "start_time": "33.93", + "end_time": "34.259" + }, + { + "id": 75, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "to" + } + ], + "start_time": "34.27", + "end_time": "34.389" + }, + { + "id": 76, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.997", + "content": "describe" + } + ], + "start_time": "34.4", + "end_time": "34.84" + }, + { + "id": 77, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.764", + "content": "many" + } + ], + "start_time": "34.849", + "end_time": "35.099" + }, + { + "id": 78, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.691", + "content": "of" + } + ], + "start_time": "35.11", + "end_time": "35.13" + }, + { + "id": 79, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.753", + "content": "the" + } + ], + "start_time": "35.139", + "end_time": "35.159" + }, + { + "id": 80, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.731", + "content": "funds" + } + ], + "start_time": "35.569", + "end_time": "36.869" + }, + { + "id": 81, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.581", + "content": "of" + } + ], + "start_time": "36.959", + "end_time": "36.979" + }, + { + "id": 82, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.288", + "content": "production" + } + ], + "start_time": "37.08", + "end_time": "37.29" + }, + { + "id": 83, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.684", + "content": "tokens" + } + ], + "start_time": "37.689", + "end_time": "37.9" + }, + { + "id": 84, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.96", + "content": "have" + } + ], + "start_time": "37.909", + "end_time": "38.09" + }, + { + "id": 85, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "been" + } + ], + "start_time": "38.099", + "end_time": "38.24" + }, + { + "id": 86, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.991", + "content": "created" + } + ], + "start_time": "38.25", + "end_time": "38.959" + }, + { + "id": 87, + "type": "punctuation", + "alternatives": [ + { + "confidence": "0.0", + "content": "." + } + ] + }, + { + "id": 88, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.756", + "content": "Cryptocurrency" + } + ], + "start_time": "38.97", + "end_time": "39.77" + }, + { + "id": 89, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "are" + } + ], + "start_time": "39.779", + "end_time": "39.83" + }, + { + "id": 90, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "not" + } + ], + "start_time": "39.84", + "end_time": "40.0" + }, + { + "id": 91, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "considered" + } + ], + "start_time": "40.009", + "end_time": "40.58" + }, + { + "id": 92, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.858", + "content": "currency" + } + ], + "start_time": "40.59", + "end_time": "41.159" + }, + { + "id": 93, + "type": "punctuation", + "alternatives": [ + { + "confidence": "0.0", + "content": "," + } + ] + }, + { + "id": 94, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.607", + "content": "traditional" + } + ], + "start_time": "41.259", + "end_time": "41.759" + }, + { + "id": 95, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.855", + "content": "things" + } + ], + "start_time": "41.88", + "end_time": "42.099" + }, + { + "id": 96, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.874", + "content": "and" + } + ], + "start_time": "42.63", + "end_time": "42.759" + }, + { + "id": 97, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.944", + "content": "very" + } + ], + "start_time": "42.77", + "end_time": "43.169" + }, + { + "id": 98, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.911", + "content": "tr" + } + ], + "start_time": "43.18", + "end_time": "43.22" + }, + { + "id": 99, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "treatments" + } + ], + "start_time": "44.54", + "end_time": "45.639" + }, + { + "id": 100, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.94", + "content": "have" + } + ], + "start_time": "46.099", + "end_time": "46.459" + }, + { + "id": 101, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.93", + "content": "implied" + } + ], + "start_time": "46.47", + "end_time": "47.02" + }, + { + "id": 102, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.872", + "content": "to" + } + ], + "start_time": "47.029", + "end_time": "47.06" + }, + { + "id": 103, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.002", + "content": "the" + } + ], + "start_time": "47.389", + "end_time": "47.409" + }, + { + "id": 104, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.605", + "content": "area" + } + ], + "start_time": "47.419", + "end_time": "47.639" + }, + { + "id": 105, + "type": "punctuation", + "alternatives": [ + { + "confidence": "0.0", + "content": "." + } + ] + }, + { + "id": 106, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.997", + "content": "Various" + } + ], + "start_time": "48.689", + "end_time": "49.209" + }, + { + "id": 107, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "jurisdictions" + } + ], + "start_time": "49.22", + "end_time": "50.13" + }, + { + "id": 108, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.985", + "content": "include" + } + ], + "start_time": "50.139", + "end_time": "50.52" + }, + { + "id": 109, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.928", + "content": "the" + } + ], + "start_time": "50.529", + "end_time": "50.59" + }, + { + "id": 110, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "classification" + } + ], + "start_time": "50.599", + "end_time": "51.27" + }, + { + "id": 111, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.934", + "content": "of" + } + ], + "start_time": "51.279", + "end_time": "51.389" + }, + { + "id": 112, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.953", + "content": "commodities" + } + ], + "start_time": "51.4", + "end_time": "52.0" + }, + { + "id": 113, + "type": "punctuation", + "alternatives": [ + { + "confidence": "0.0", + "content": "," + } + ] + }, + { + "id": 114, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "securities" + } + ], + "start_time": "52.009", + "end_time": "52.93" + }, + { + "id": 115, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.32", + "content": "and" + } + ], + "start_time": "53.18", + "end_time": "53.229" + }, + { + "id": 116, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.573", + "content": "currencies" + } + ], + "start_time": "53.24", + "end_time": "54.04" + }, + { + "id": 117, + "type": "punctuation", + "alternatives": [ + { + "confidence": "0.0", + "content": "." + } + ] + }, + { + "id": 118, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.283", + "content": "Cryptocurrency" + } + ], + "start_time": "54.22", + "end_time": "55.08" + }, + { + "id": 119, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "are" + } + ], + "start_time": "55.09", + "end_time": "55.18" + }, + { + "id": 120, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "generally" + } + ], + "start_time": "55.189", + "end_time": "55.7" + }, + { + "id": 121, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.959", + "content": "viewed" + } + ], + "start_time": "55.709", + "end_time": "56.069" + }, + { + "id": 122, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.77", + "content": "as" + } + ], + "start_time": "56.08", + "end_time": "56.2" + }, + { + "id": 123, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.43", + "content": "this" + } + ], + "start_time": "56.369", + "end_time": "56.529" + }, + { + "id": 124, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.61", + "content": "cost" + } + ], + "start_time": "56.88", + "end_time": "57.02" + }, + { + "id": 125, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.989", + "content": "cost" + } + ], + "start_time": "57.13", + "end_time": "57.56" + }, + { + "id": 126, + "type": "punctuation", + "alternatives": [ + { + "confidence": "0.0", + "content": "." + } + ] + }, + { + "id": 127, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.816", + "content": "In" + } + ], + "start_time": "57.569", + "end_time": "57.65" + }, + { + "id": 128, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "practice" + } + ], + "start_time": "57.659", + "end_time": "58.459" + }, + { + "id": 129, + "type": "punctuation", + "alternatives": [ + { + "confidence": "0.0", + "content": "," + } + ] + }, + { + "id": 130, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.997", + "content": "some" + } + ], + "start_time": "58.729", + "end_time": "59.11" + }, + { + "id": 131, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.994", + "content": "crypto" + } + ], + "start_time": "59.119", + "end_time": "59.45" + }, + { + "id": 132, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.979", + "content": "schemes" + } + ], + "start_time": "59.669", + "end_time": "60.349" + }, + { + "id": 133, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.917", + "content": "use" + } + ], + "start_time": "61.279", + "end_time": "61.68" + }, + { + "id": 134, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.723", + "content": "the" + } + ], + "start_time": "61.74", + "end_time": "61.759" + }, + { + "id": 135, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.531", + "content": "data" + } + ], + "start_time": "61.779", + "end_time": "61.819" + }, + { + "id": 136, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.997", + "content": "to" + } + ], + "start_time": "62.11", + "end_time": "62.43" + }, + { + "id": 137, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "maintain" + } + ], + "start_time": "62.439", + "end_time": "62.99" + }, + { + "id": 138, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.996", + "content": "the" + } + ], + "start_time": "63.0", + "end_time": "63.099" + }, + { + "id": 139, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.963", + "content": "Cryptocurrency" + } + ], + "start_time": "63.11", + "end_time": "64.019" + }, + { + "id": 140, + "type": "punctuation", + "alternatives": [ + { + "confidence": "0.0", + "content": "." + } + ] + }, + { + "id": 141, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.992", + "content": "Thank" + } + ], + "start_time": "65.47", + "end_time": "65.779" + }, + { + "id": 142, + "type": "pronunciation", + "alternatives": [ + { + "confidence": "0.967", + "content": "you" + } + ], + "start_time": "65.79", + "end_time": "65.93" + }, + { + "id": 143, + "type": "punctuation", + "alternatives": [ + { + "confidence": "0.0", + "content": "." + } + ] + } + ], + "audio_segments": [ + { + "id": 0, + "transcript": "The currency Cryptocurrency or crypto is a digital currency designed to work through a computer network that is not the one of any central authority such as government or bank job hold or maintain it", + "start_time": "3.859", + "end_time": "17.729", + "items": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34 + ] + }, + { + "id": 1, + "transcript": "individual coin own records. A to digital ledger which is computerized database, using strong Cryptocurrency to secure transaction codes control the creation of additional coins to verify the transfer co ownership.", + "start_time": "18.329", + "end_time": "32.069", + "items": [ + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68 + ] + }, + { + "id": 2, + "transcript": "Despite the term that has come to describe many of the funds of production tokens have been created. Cryptocurrency are not considered currency, traditional things and very tr treatments have implied to the area.", + "start_time": "32.65", + "end_time": "47.909", + "items": [ + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105 + ] + }, + { + "id": 3, + "transcript": "Various jurisdictions include the classification of commodities, securities and currencies. Cryptocurrency are generally viewed as this cost cost. In practice, some crypto schemes", + "start_time": "48.68", + "end_time": "60.43", + "items": [ + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "id": 4, + "transcript": "use the data to maintain the Cryptocurrency.", + "start_time": "61.11", + "end_time": "64.12", + "items": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140 + ] + }, + { + "id": 5, + "transcript": "Thank you.", + "start_time": "65.36", + "end_time": "66.04", + "items": [ + 141, + 142, + 143 + ] + } + ] + } +} \ No newline at end of file diff --git a/prompt-reminder.json b/prompt-reminder.json new file mode 100644 index 0000000..4aaf737 --- /dev/null +++ b/prompt-reminder.json @@ -0,0 +1,14 @@ +[ + { + "remindAt": "2023-05-12T14:00:00Z", + "description": "alice@gmail.com and bob@gmail.com: Sync up at 2 PM to discuss progress" + }, + { + "remindAt": "2023-05-12T15:00:00Z", + "description": "alice@gmail.com and bob@gmail.com: Final run-through for presentation" + }, + { + "remindAt": null, + "description": "alice@gmail.com and bob@gmail.com: Finalize budget by Friday" + } +] \ No newline at end of file diff --git a/run-kurento-media-server.sh b/run-kurento-media-server.sh new file mode 100755 index 0000000..93e0ec4 --- /dev/null +++ b/run-kurento-media-server.sh @@ -0,0 +1,10 @@ +docker run -d \ + -v /tmp/recordings:/tmp/recordings \ + -p 8888:8888/tcp \ + -p 5000-5050:5000-5050/udp \ + -e KMS_MIN_PORT=5000 \ + -e KMS_MAX_PORT=5050 \ + -e KMS_STUN_IP=stun.l.google.com \ + -e KMS_STUN_PORT=19302 \ + -e KMS_TURN_URL=2541d7b4503e7956a66ab39e:fInHfabub1gAFe74@188.245.177.56:80 \ + kurento/kurento-media-server:7.1.0 \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 5c8e1d4..79b89eb 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1 +1 @@ -rootProject.name = 'open-messanger' +rootProject.name = 'open-messenger' diff --git a/src/main/kotlin/io/openfuture/openmessanger/OpenMessangerApplication.kt b/src/main/kotlin/io/openfuture/openmessanger/OpenMessangerApplication.kt deleted file mode 100644 index fa7231e..0000000 --- a/src/main/kotlin/io/openfuture/openmessanger/OpenMessangerApplication.kt +++ /dev/null @@ -1,12 +0,0 @@ -package io.openfuture.openmessanger - -import org.springframework.boot.SpringApplication -import org.springframework.boot.autoconfigure.SpringBootApplication - -@SpringBootApplication -class OpenMessangerApplication - -fun main(args: Array) { - SpringApplication.run(OpenMessangerApplication::class.java, *args) -} - diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/ChatParticipantRepository.kt b/src/main/kotlin/io/openfuture/openmessanger/repository/ChatParticipantRepository.kt deleted file mode 100644 index b0d6d5c..0000000 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/ChatParticipantRepository.kt +++ /dev/null @@ -1,8 +0,0 @@ -package io.openfuture.openmessanger.repository - -import io.openfuture.openmessanger.repository.entity.ChatParticipant -import org.springframework.data.jpa.repository.JpaRepository - -interface ChatParticipantRepository : JpaRepository { - fun findAllByChatId(chat: Int?): List? -} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/MessageAttachment.kt b/src/main/kotlin/io/openfuture/openmessanger/repository/entity/MessageAttachment.kt deleted file mode 100644 index 3c2b7e7..0000000 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/MessageAttachment.kt +++ /dev/null @@ -1,14 +0,0 @@ -package io.openfuture.openmessanger.repository.entity - -import jakarta.persistence.* - -@Entity -@Table(name = "message_attachment") -class MessageAttachment(@Column(name = "attachment_id") var attachmentId: Int?, - @Column(name = "message_id") var messageId: Int?) { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - val id: Int? = null - -} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/PrivateChatService.kt b/src/main/kotlin/io/openfuture/openmessanger/service/PrivateChatService.kt deleted file mode 100644 index 97ffbfd..0000000 --- a/src/main/kotlin/io/openfuture/openmessanger/service/PrivateChatService.kt +++ /dev/null @@ -1,7 +0,0 @@ -package io.openfuture.openmessanger.service - -import io.openfuture.openmessanger.repository.entity.ChatParticipant - -interface PrivateChatService { - fun getOtherUser(username: String, chatId: Int?): ChatParticipant? -} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/OpenMessengerApplication.kt b/src/main/kotlin/io/openfuture/openmessenger/OpenMessengerApplication.kt new file mode 100644 index 0000000..9e0e9ab --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/OpenMessengerApplication.kt @@ -0,0 +1,25 @@ +package io.openfuture.openmessenger + +import io.openfuture.openmessenger.service.AssistantService +import org.springframework.boot.CommandLineRunner +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.domain.EntityScan +import org.springframework.context.annotation.Bean +import org.springframework.web.servlet.config.annotation.CorsRegistry +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer + + +@EntityScan("io.openfuture.openmessenger.repository.entity") +@SpringBootApplication +class OpenMessengerApplication(val assistantService: AssistantService): CommandLineRunner { + override fun run(vararg args: String?) { +// val generateNotes = aiProcessor.generateNotes(AiRequest(33, false, LocalDateTime.now().minusDays(13), LocalDateTime.now())) +// print(generateNotes) + } +} + +fun main(args: Array) { + SpringApplication.run(OpenMessengerApplication::class.java, *args) +} + diff --git a/src/main/kotlin/io/openfuture/openmessanger/assistant/gemini/GeminiHttpConfiguration.kt b/src/main/kotlin/io/openfuture/openmessenger/assistant/gemini/GeminiHttpConfiguration.kt similarity index 93% rename from src/main/kotlin/io/openfuture/openmessanger/assistant/gemini/GeminiHttpConfiguration.kt rename to src/main/kotlin/io/openfuture/openmessenger/assistant/gemini/GeminiHttpConfiguration.kt index 2882ffa..e6d74f5 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/assistant/gemini/GeminiHttpConfiguration.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/assistant/gemini/GeminiHttpConfiguration.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.assistant.gemini +package io.openfuture.openmessenger.assistant.gemini import org.springframework.beans.factory.annotation.Value import org.springframework.context.annotation.Bean diff --git a/src/main/kotlin/io/openfuture/openmessanger/assistant/gemini/GeminiResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/assistant/gemini/GeminiResponse.kt similarity index 89% rename from src/main/kotlin/io/openfuture/openmessanger/assistant/gemini/GeminiResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/assistant/gemini/GeminiResponse.kt index 0ce850c..0da0cca 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/assistant/gemini/GeminiResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/assistant/gemini/GeminiResponse.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.assistant.gemini +package io.openfuture.openmessenger.assistant.gemini data class GeminiResponse( val candidates: List? = null diff --git a/src/main/kotlin/io/openfuture/openmessanger/assistant/gemini/GeminiService.kt b/src/main/kotlin/io/openfuture/openmessenger/assistant/gemini/GeminiService.kt similarity index 69% rename from src/main/kotlin/io/openfuture/openmessanger/assistant/gemini/GeminiService.kt rename to src/main/kotlin/io/openfuture/openmessenger/assistant/gemini/GeminiService.kt index d6c79c4..41d77eb 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/assistant/gemini/GeminiService.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/assistant/gemini/GeminiService.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.assistant.gemini +package io.openfuture.openmessenger.assistant.gemini import org.springframework.beans.factory.annotation.Qualifier import org.springframework.beans.factory.annotation.Value @@ -16,7 +16,21 @@ class GeminiService ( ) { fun chat(input: String?): String? { - val jsonPayload = String.format("{\"contents\":[{\"role\": \"user\", \"parts\":[{\"text\": \"%s\"}]}]}", input) + val jsonPayload = """ + { + "contents": [ + { + "role": "user", + "parts": [ + { + "text": "$input" + } + ] + } + ] + } + """.trimIndent() + val headers = HttpHeaders() headers.contentType = MediaType.APPLICATION_JSON val requestEntity = HttpEntity(jsonPayload, headers) diff --git a/src/main/kotlin/io/openfuture/openmessenger/assistant/model/BaseModel.kt b/src/main/kotlin/io/openfuture/openmessenger/assistant/model/BaseModel.kt new file mode 100644 index 0000000..ac41de5 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/assistant/model/BaseModel.kt @@ -0,0 +1,14 @@ +package io.openfuture.openmessenger.assistant.model + +import java.time.LocalDateTime + +interface BaseModel { + val chatId: Int? + val groupChatId: Int? + val members: List? + val recipient: String? + val generatedAt: LocalDateTime + val version: Int + val startTime: LocalDateTime + val endTime: LocalDateTime +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/assistant/model/ConversationNotes.kt b/src/main/kotlin/io/openfuture/openmessenger/assistant/model/ConversationNotes.kt new file mode 100644 index 0000000..3fdbbab --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/assistant/model/ConversationNotes.kt @@ -0,0 +1,15 @@ +package io.openfuture.openmessenger.assistant.model + +import java.time.LocalDateTime + +data class ConversationNotes( + override val chatId: Int?, + override val groupChatId: Int?, + override val members: List?, + override val recipient: String?, + override val generatedAt: LocalDateTime, + override val version: Int, + override val startTime: LocalDateTime, + override val endTime: LocalDateTime, + val notes: List +): BaseModel diff --git a/src/main/kotlin/io/openfuture/openmessenger/assistant/model/Reminder.kt b/src/main/kotlin/io/openfuture/openmessenger/assistant/model/Reminder.kt new file mode 100644 index 0000000..a2811a6 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/assistant/model/Reminder.kt @@ -0,0 +1,21 @@ +package io.openfuture.openmessenger.assistant.model + +import java.time.LocalDateTime + +data class Reminder( + override val chatId: Int?, + override val groupChatId: Int?, + override val members: List?, + override val recipient: String?, + override val generatedAt: LocalDateTime, + override val version: Int, + override val startTime: LocalDateTime, + override val endTime: LocalDateTime, + + val reminders: List +): BaseModel + +data class ReminderItem( + val remindAt: LocalDateTime?, + val description: String? +) \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/assistant/model/Todos.kt b/src/main/kotlin/io/openfuture/openmessenger/assistant/model/Todos.kt new file mode 100644 index 0000000..cd18322 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/assistant/model/Todos.kt @@ -0,0 +1,23 @@ +package io.openfuture.openmessenger.assistant.model + +import java.time.LocalDateTime + +data class Todos( + override val chatId: Int?, + override val groupChatId: Int?, + override val members: List?, + override val recipient: String?, + override val generatedAt: LocalDateTime, + override val version: Int, + override val startTime: LocalDateTime, + override val endTime: LocalDateTime, + + val todos: List +): BaseModel + +data class Todo( + val executor: String?, + val description: String?, + val dueDate: LocalDateTime?, + val context: String? +) \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/configuration/AppConfig.kt b/src/main/kotlin/io/openfuture/openmessenger/configuration/AppConfig.kt similarity index 93% rename from src/main/kotlin/io/openfuture/openmessanger/configuration/AppConfig.kt rename to src/main/kotlin/io/openfuture/openmessenger/configuration/AppConfig.kt index a560b1a..e65eb1d 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/configuration/AppConfig.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/configuration/AppConfig.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.configuration +package io.openfuture.openmessenger.configuration import com.amazonaws.auth.AWSStaticCredentialsProvider import com.amazonaws.auth.BasicAWSCredentials diff --git a/src/main/kotlin/io/openfuture/openmessanger/configuration/AwsConfig.kt b/src/main/kotlin/io/openfuture/openmessenger/configuration/AwsConfig.kt similarity index 80% rename from src/main/kotlin/io/openfuture/openmessanger/configuration/AwsConfig.kt rename to src/main/kotlin/io/openfuture/openmessenger/configuration/AwsConfig.kt index 9f233ea..4a54897 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/configuration/AwsConfig.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/configuration/AwsConfig.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.configuration +package io.openfuture.openmessenger.configuration import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.stereotype.Component @@ -10,6 +10,8 @@ data class AwsConfig( var secretKey: String? = null, var region: String? = null, var attachmentsBucket: String? = null, + var recordingsBucket: String? = null, + var transcriptsBucket: String? = null, var cognito: Cognito = Cognito() ) diff --git a/src/main/kotlin/io/openfuture/openmessanger/configuration/AwsS3Config.kt b/src/main/kotlin/io/openfuture/openmessenger/configuration/AwsS3Config.kt similarity index 94% rename from src/main/kotlin/io/openfuture/openmessanger/configuration/AwsS3Config.kt rename to src/main/kotlin/io/openfuture/openmessenger/configuration/AwsS3Config.kt index b017654..8a520a7 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/configuration/AwsS3Config.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/configuration/AwsS3Config.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.configuration +package io.openfuture.openmessenger.configuration import com.amazonaws.auth.AWSCredentials import com.amazonaws.auth.AWSStaticCredentialsProvider diff --git a/src/main/kotlin/io/openfuture/openmessanger/configuration/SecurityConfig.kt b/src/main/kotlin/io/openfuture/openmessenger/configuration/SecurityConfig.kt similarity index 69% rename from src/main/kotlin/io/openfuture/openmessanger/configuration/SecurityConfig.kt rename to src/main/kotlin/io/openfuture/openmessenger/configuration/SecurityConfig.kt index 116e27b..7fa69e9 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/configuration/SecurityConfig.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/configuration/SecurityConfig.kt @@ -1,19 +1,22 @@ -package io.openfuture.openmessanger.configuration +package io.openfuture.openmessenger.configuration -import io.openfuture.openmessanger.security.AwsCognitoTokenFilter -import io.openfuture.openmessanger.security.CognitoAuthenticationProvider +import io.openfuture.openmessenger.security.AwsCognitoTokenFilter +import io.openfuture.openmessenger.security.CognitoAuthenticationProvider +import jakarta.servlet.http.HttpServletRequest import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import org.springframework.security.config.Customizer import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder import org.springframework.security.config.annotation.web.builders.HttpSecurity import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity -import org.springframework.security.config.annotation.web.configurers.AuthorizeHttpRequestsConfigurer.AuthorizationManagerRequestMatcherRegistry import org.springframework.security.config.annotation.web.configurers.CsrfConfigurer import org.springframework.security.config.annotation.web.configurers.SessionManagementConfigurer import org.springframework.security.config.http.SessionCreationPolicy import org.springframework.security.web.SecurityFilterChain import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter +import org.springframework.web.cors.CorsConfiguration +import org.springframework.web.cors.CorsConfigurationSource +import org.springframework.web.cors.UrlBasedCorsConfigurationSource @Configuration @EnableWebSecurity @@ -33,7 +36,13 @@ class SecurityConfig( .authorizeHttpRequests { it.requestMatchers("/api/v1/public/login").permitAll() it.requestMatchers("/api/v1/public/signup").permitAll() - //.requestMatchers("/**").permitAll() + it.requestMatchers("/api/v1/attachments/download/**").permitAll() + it.requestMatchers("/*").permitAll() + it.requestMatchers("/webjars/**").permitAll() + it.requestMatchers("/js/*").permitAll() + it.requestMatchers("/img/*").permitAll() + it.requestMatchers("/css/*").permitAll() + it.requestMatchers("/video/*").permitAll() it.anyRequest().authenticated() } .addFilterBefore( @@ -41,7 +50,9 @@ class SecurityConfig( "/api/**", authenticationManager, "/api/v1/public/login", - "/api/v1/public/signup" + "/api/v1/public/signup", + "/api/v1/attachments/download/**", + listOf("/*", "/webjars/**", "/js/*", "/img/*", "/css/*", "/video/*") ), UsernamePasswordAuthenticationFilter::class.java ) diff --git a/src/main/kotlin/io/openfuture/openmessenger/configuration/TranscribeConfig.kt b/src/main/kotlin/io/openfuture/openmessenger/configuration/TranscribeConfig.kt new file mode 100644 index 0000000..10db6a9 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/configuration/TranscribeConfig.kt @@ -0,0 +1,25 @@ +package io.openfuture.openmessenger.configuration + +import com.amazonaws.auth.AWSCredentials +import com.amazonaws.auth.AWSStaticCredentialsProvider +import com.amazonaws.regions.Regions +import com.amazonaws.services.transcribe.AmazonTranscribe +import com.amazonaws.services.transcribe.AmazonTranscribeClientBuilder +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +@Configuration +class TranscribeConfig( + private val awsCredentials: AWSCredentials +) { + + @Bean + fun transcribeClient(): AmazonTranscribe { + return AmazonTranscribeClientBuilder + .standard() + .withRegion(Regions.US_EAST_2) + .withCredentials(AWSStaticCredentialsProvider(awsCredentials)) + .build() + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/configuration/WebSocketConfig.kt b/src/main/kotlin/io/openfuture/openmessenger/configuration/WebSocketConfig.kt similarity index 94% rename from src/main/kotlin/io/openfuture/openmessanger/configuration/WebSocketConfig.kt rename to src/main/kotlin/io/openfuture/openmessenger/configuration/WebSocketConfig.kt index a190e48..9b014b0 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/configuration/WebSocketConfig.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/configuration/WebSocketConfig.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.configuration +package io.openfuture.openmessenger.configuration import org.springframework.context.annotation.Configuration import org.springframework.messaging.simp.config.MessageBrokerRegistry diff --git a/src/main/kotlin/io/openfuture/openmessanger/configuration/WebSocketRabbitConfig.kt b/src/main/kotlin/io/openfuture/openmessenger/configuration/WebSocketRabbitConfig.kt similarity index 96% rename from src/main/kotlin/io/openfuture/openmessanger/configuration/WebSocketRabbitConfig.kt rename to src/main/kotlin/io/openfuture/openmessenger/configuration/WebSocketRabbitConfig.kt index 0218020..5694449 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/configuration/WebSocketRabbitConfig.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/configuration/WebSocketRabbitConfig.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.configuration +package io.openfuture.openmessenger.configuration import org.springframework.messaging.simp.config.MessageBrokerRegistry import org.springframework.web.socket.config.annotation.StompEndpointRegistry diff --git a/src/main/kotlin/io/openfuture/openmessanger/domain/User.kt b/src/main/kotlin/io/openfuture/openmessenger/domain/User.kt similarity index 80% rename from src/main/kotlin/io/openfuture/openmessanger/domain/User.kt rename to src/main/kotlin/io/openfuture/openmessenger/domain/User.kt index 9b9ec7c..7dbecd0 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/domain/User.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/domain/User.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.domain +package io.openfuture.openmessenger.domain import lombok.AllArgsConstructor import lombok.Data diff --git a/src/main/kotlin/io/openfuture/openmessanger/domain/enums/CognitoAttributesEnum.kt b/src/main/kotlin/io/openfuture/openmessenger/domain/enums/CognitoAttributesEnum.kt similarity index 92% rename from src/main/kotlin/io/openfuture/openmessanger/domain/enums/CognitoAttributesEnum.kt rename to src/main/kotlin/io/openfuture/openmessenger/domain/enums/CognitoAttributesEnum.kt index 5124d10..bd76661 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/domain/enums/CognitoAttributesEnum.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/domain/enums/CognitoAttributesEnum.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.domain.enums +package io.openfuture.openmessenger.domain.enums enum class CognitoAttributesEnum(val values: String) { USERNAME("USERNAME"), diff --git a/src/main/kotlin/io/openfuture/openmessanger/exception/BaseExceptionHandler.kt b/src/main/kotlin/io/openfuture/openmessenger/exception/BaseExceptionHandler.kt similarity index 95% rename from src/main/kotlin/io/openfuture/openmessanger/exception/BaseExceptionHandler.kt rename to src/main/kotlin/io/openfuture/openmessenger/exception/BaseExceptionHandler.kt index 7ff199d..c26eac0 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/exception/BaseExceptionHandler.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/exception/BaseExceptionHandler.kt @@ -1,8 +1,8 @@ -package io.openfuture.openmessanger.exception +package io.openfuture.openmessenger.exception import com.amazonaws.services.cognitoidp.model.InternalErrorException import com.amazonaws.services.cognitoidp.model.NotAuthorizedException -import io.openfuture.openmessanger.web.response.BaseResponse +import io.openfuture.openmessenger.web.response.BaseResponse import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.http.HttpStatus diff --git a/src/main/kotlin/io/openfuture/openmessanger/exception/FailedAuthenticationException.kt b/src/main/kotlin/io/openfuture/openmessenger/exception/FailedAuthenticationException.kt similarity index 82% rename from src/main/kotlin/io/openfuture/openmessanger/exception/FailedAuthenticationException.kt rename to src/main/kotlin/io/openfuture/openmessenger/exception/FailedAuthenticationException.kt index 992a320..d64c5e2 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/exception/FailedAuthenticationException.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/exception/FailedAuthenticationException.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.exception +package io.openfuture.openmessenger.exception class FailedAuthenticationException : ServiceException { constructor() : super() diff --git a/src/main/kotlin/io/openfuture/openmessanger/exception/FirstTimeLoginException.kt b/src/main/kotlin/io/openfuture/openmessenger/exception/FirstTimeLoginException.kt similarity index 82% rename from src/main/kotlin/io/openfuture/openmessanger/exception/FirstTimeLoginException.kt rename to src/main/kotlin/io/openfuture/openmessenger/exception/FirstTimeLoginException.kt index e86c245..94a172e 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/exception/FirstTimeLoginException.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/exception/FirstTimeLoginException.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.exception +package io.openfuture.openmessenger.exception class FirstTimeLoginException : ServiceException { constructor() : super() diff --git a/src/main/kotlin/io/openfuture/openmessanger/exception/InvalidParameterException.kt b/src/main/kotlin/io/openfuture/openmessenger/exception/InvalidParameterException.kt similarity index 82% rename from src/main/kotlin/io/openfuture/openmessanger/exception/InvalidParameterException.kt rename to src/main/kotlin/io/openfuture/openmessenger/exception/InvalidParameterException.kt index 6f0c061..80ec6ae 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/exception/InvalidParameterException.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/exception/InvalidParameterException.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.exception +package io.openfuture.openmessenger.exception class InvalidParameterException : ServiceException { constructor() : super() diff --git a/src/main/kotlin/io/openfuture/openmessanger/exception/InvalidPasswordException.kt b/src/main/kotlin/io/openfuture/openmessenger/exception/InvalidPasswordException.kt similarity index 81% rename from src/main/kotlin/io/openfuture/openmessanger/exception/InvalidPasswordException.kt rename to src/main/kotlin/io/openfuture/openmessenger/exception/InvalidPasswordException.kt index 4ac36af..4e63d21 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/exception/InvalidPasswordException.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/exception/InvalidPasswordException.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.exception +package io.openfuture.openmessenger.exception class InvalidPasswordException : ServiceException { constructor() diff --git a/src/main/kotlin/io/openfuture/openmessanger/exception/ServiceException.kt b/src/main/kotlin/io/openfuture/openmessenger/exception/ServiceException.kt similarity index 85% rename from src/main/kotlin/io/openfuture/openmessanger/exception/ServiceException.kt rename to src/main/kotlin/io/openfuture/openmessenger/exception/ServiceException.kt index 1ac0705..fcc2caf 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/exception/ServiceException.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/exception/ServiceException.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.exception +package io.openfuture.openmessenger.exception open class ServiceException : RuntimeException { constructor() : super() diff --git a/src/main/kotlin/io/openfuture/openmessanger/exception/UserNotFoundException.kt b/src/main/kotlin/io/openfuture/openmessenger/exception/UserNotFoundException.kt similarity index 82% rename from src/main/kotlin/io/openfuture/openmessanger/exception/UserNotFoundException.kt rename to src/main/kotlin/io/openfuture/openmessenger/exception/UserNotFoundException.kt index 45b234e..c024e4d 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/exception/UserNotFoundException.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/exception/UserNotFoundException.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.exception +package io.openfuture.openmessenger.exception class UserNotFoundException : ServiceException { constructor() : super() diff --git a/src/main/kotlin/io/openfuture/openmessanger/exception/UsernameExistsException.kt b/src/main/kotlin/io/openfuture/openmessenger/exception/UsernameExistsException.kt similarity index 82% rename from src/main/kotlin/io/openfuture/openmessanger/exception/UsernameExistsException.kt rename to src/main/kotlin/io/openfuture/openmessenger/exception/UsernameExistsException.kt index 1c92faa..fd0a1f2 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/exception/UsernameExistsException.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/exception/UsernameExistsException.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.exception +package io.openfuture.openmessenger.exception class UsernameExistsException : ServiceException { constructor() : super() diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/CallMediaPipeline.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/CallMediaPipeline.kt new file mode 100644 index 0000000..c952c7d --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/CallMediaPipeline.kt @@ -0,0 +1,102 @@ +package io.openfuture.openmessenger.kurento + +import io.openfuture.openmessenger.exception.BaseExceptionHandler +import org.kurento.client.KurentoClient +import org.kurento.client.MediaPipeline +import org.kurento.client.RecorderEndpoint +import org.kurento.client.WebRtcEndpoint +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.io.File +import java.text.SimpleDateFormat +import java.util.* + +class CallMediaPipeline(kurento: KurentoClient, appointmentId: Long, from: String, to: String) { + private val RECORDING_PATH = "file:/" + + private val pipeline: MediaPipeline + private val webRtcCaller: WebRtcEndpoint + private val webRtcCallee: WebRtcEndpoint + private val recorderCaller: RecorderEndpoint + private val recorderCallee: RecorderEndpoint + + init { + val directory = mkdirs(appointmentId) + + pipeline = kurento.createMediaPipeline() + webRtcCaller = WebRtcEndpoint.Builder(pipeline).build() + webRtcCallee = WebRtcEndpoint.Builder(pipeline).build() + + recorderCaller = RecorderEndpoint.Builder( + pipeline, + RECORDING_PATH + directory + DATE_FORMAT_VIDEO_FILE.format(Date()) + "_doctor_" + from + RECORDING_EXT + ) + .build() + recorderCallee = RecorderEndpoint.Builder( + pipeline, + RECORDING_PATH + directory + DATE_FORMAT_VIDEO_FILE.format(Date()) + "_patient_" + to + RECORDING_EXT + ) + .build() + + webRtcCaller.connect(webRtcCallee) + webRtcCaller.connect(recorderCaller) + + webRtcCallee.connect(webRtcCaller) + webRtcCallee.connect(recorderCallee) + } + + private fun mkdirs(appointmentId: Long): String { + val directory = String.format(VIDEO_DIR, appointmentId) + + val file = File(directory) + if (!file.exists()) { + mkdirs(directory) + } + + return directory + } + + fun record() { + recorderCaller.record() + recorderCallee.record() + } + + fun generateSdpAnswerForCaller(sdpOffer: String?): String { + return webRtcCaller.processOffer(sdpOffer) + } + + fun generateSdpAnswerForCallee(sdpOffer: String?): String { + return webRtcCallee.processOffer(sdpOffer) + } + + fun getPipeline(): MediaPipeline { + return pipeline + } + + val callerWebRtcEp: WebRtcEndpoint + get() = webRtcCaller + + val calleeWebRtcEp: WebRtcEndpoint + get() = webRtcCallee + + private fun mkdirs(directory: String) { + try { + val p = Runtime.getRuntime().exec("mkdir -m 777 $directory") + + p.waitFor() + p.destroy() + } catch (e: Exception) { + log.error("Can't execute command of directory creation") + } + } + + companion object { + private val DATE_FORMAT_VIDEO_FILE = SimpleDateFormat("yyyy-MM-dd_HH-mm-ss") + private val DATE_FORMAT_VIDEO_DIR = SimpleDateFormat("yyyy-MM-dd") + private const val RECORDING_EXT = ".webm" + + private val log: Logger = LoggerFactory.getLogger(BaseExceptionHandler::class.java) + private val VIDEO_DIR = "/blobs/medonline/video/" + DATE_FORMAT_VIDEO_DIR.format(Date()) + "_%s/" + } + +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/HelloWorldHandler.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/HelloWorldHandler.kt new file mode 100644 index 0000000..4dcdb9b --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/HelloWorldHandler.kt @@ -0,0 +1,423 @@ +package io.openfuture.openmessenger.kurento + +import com.google.gson.Gson +import com.google.gson.GsonBuilder +import com.google.gson.JsonObject +import org.kurento.client.BaseRtpEndpoint +import org.kurento.client.IceCandidate +import org.kurento.client.KurentoClient +import org.kurento.client.WebRtcEndpoint +import org.kurento.jsonrpc.JsonUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.web.socket.CloseStatus +import org.springframework.web.socket.TextMessage +import org.springframework.web.socket.WebSocketSession +import org.springframework.web.socket.handler.TextWebSocketHandler +import java.io.IOException +import java.util.concurrent.ConcurrentHashMap + +class HelloWorldHandler : TextWebSocketHandler() { + private val users = ConcurrentHashMap() + + @Autowired + private val kurento: KurentoClient? = null + + @Throws(Exception::class) + override fun afterConnectionEstablished(session: WebSocketSession) { + log.info( + "[HelloWorldHandler::afterConnectionEstablished] New WebSocket connection, sessionId: {}", + session.id + ) + } + + @Throws(Exception::class) + override fun afterConnectionClosed( + session: WebSocketSession, + status: CloseStatus + ) { + if (!status.equalsCode(CloseStatus.NORMAL)) { + log.warn( + "[HelloWorldHandler::afterConnectionClosed] status: {}, sessionId: {}", + status, session.id + ) + } + + stop(session) + } + + @Throws(Exception::class) + override fun handleTextMessage( + session: WebSocketSession, + message: TextMessage + ) { + val sessionId = session.id + val jsonMessage = gson.fromJson( + message.payload, + JsonObject::class.java + ) + + log.info( + "[HelloWorldHandler::handleTextMessage] message: {}, sessionId: {}", + jsonMessage, sessionId + ) + + try { + val messageId = jsonMessage["id"].asString + when (messageId) { + "PROCESS_SDP_OFFER" -> // Start: Create user session and process SDP Offer + handleProcessSdpOffer(session, jsonMessage) + + "ADD_ICE_CANDIDATE" -> handleAddIceCandidate(session, jsonMessage) + "STOP" -> handleStop(session, jsonMessage) + "ERROR" -> handleError(session, jsonMessage) + else -> // Ignore the message + log.warn( + "[HelloWorldHandler::handleTextMessage] Skip, invalid message, id: {}", + messageId + ) + } + } catch (ex: Throwable) { + log.error( + "[HelloWorldHandler::handleTextMessage] Exception: {}, sessionId: {}", + ex, sessionId + ) + sendError(session, "[Kurento] Exception: " + ex.message) + } + } + + @Throws(Exception::class) + override fun handleTransportError( + session: WebSocketSession, + exception: Throwable + ) { + log.error( + "[HelloWorldHandler::handleTransportError] Exception: {}, sessionId: {}", + exception, session.id + ) + + session.close(CloseStatus.SERVER_ERROR) + } + + @Synchronized + private fun sendMessage( + session: WebSocketSession, + message: String + ) { + log.debug("[HelloWorldHandler::sendMessage] {}", message) + + if (!session.isOpen) { + log.warn("[HelloWorldHandler::sendMessage] Skip, WebSocket session isn't open") + return + } + + val sessionId = session.id + if (!users.containsKey(sessionId)) { + log.warn( + "[HelloWorldHandler::sendMessage] Skip, unknown user, id: {}", + sessionId + ) + return + } + + try { + session.sendMessage(TextMessage(message)) + } catch (ex: IOException) { + log.error("[HelloWorldHandler::sendMessage] Exception: {}", ex.message) + } + } + + private fun sendError(session: WebSocketSession, errMsg: String) { + log.error(errMsg) + + if (users.containsKey(session.id)) { + val message = JsonObject() + message.addProperty("id", "ERROR") + message.addProperty("message", errMsg) + sendMessage(session, message.toString()) + } + } + + // PROCESS_SDP_OFFER --------------------------------------------------------- + private fun initBaseEventListeners( + session: WebSocketSession, + baseRtpEp: BaseRtpEndpoint, className: String + ) { + log.info( + "[HelloWorldHandler::initBaseEventListeners] name: {}, class: {}, sessionId: {}", + baseRtpEp.name, className, session.id + ) + + // Event: Some error happened + baseRtpEp.addErrorListener { ev -> + log.error( + "[{}::ErrorEvent] Error code {}: '{}', source: {}, timestamp: {}, tags: {}, description: {}", + className, ev.errorCode, ev.type, ev.source.name, + ev.timestampMillis, ev.tags, ev.description + ) + sendError(session, "[Kurento] " + ev.description) + stop(session) + } + + // Event: Media is flowing into this sink + baseRtpEp.addMediaFlowInStateChangedListener { ev -> + log.info( + "[{}::{}] source: {}, timestamp: {}, tags: {}, state: {}, padName: {}, mediaType: {}", + className, ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.state, ev.padName, ev.mediaType + ) + } + + // Event: Media is flowing out of this source + baseRtpEp.addMediaFlowOutStateChangedListener { ev -> + log.info( + "[{}::{}] source: {}, timestamp: {}, tags: {}, state: {}, padName: {}, mediaType: {}", + className, ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.state, ev.padName, ev.mediaType + ) + } + + // Event: [TODO write meaning of this event] + baseRtpEp.addConnectionStateChangedListener { ev -> + log.info( + "[{}::{}] source: {}, timestamp: {}, tags: {}, oldState: {}, newState: {}", + className, ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.oldState, ev.newState + ) + } + + // Event: [TODO write meaning of this event] + baseRtpEp.addMediaStateChangedListener { ev -> + log.info( + "[{}::{}] source: {}, timestamp: {}, tags: {}, oldState: {}, newState: {}", + className, ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.oldState, ev.newState + ) + } + + // Event: This element will (or will not) perform media transcoding + baseRtpEp.addMediaTranscodingStateChangedListener { ev -> + log.info( + "[{}::{}] source: {}, timestamp: {}, tags: {}, state: {}, binName: {}, mediaType: {}", + className, ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.state, ev.binName, ev.mediaType + ) + } + } + + private fun initWebRtcEventListeners( + session: WebSocketSession, + webRtcEp: WebRtcEndpoint + ) { + log.info( + "[HelloWorldHandler::initWebRtcEventListeners] name: {}, sessionId: {}", + webRtcEp.name, session.id + ) + + // Event: The ICE backend found a local candidate during Trickle ICE + webRtcEp.addIceCandidateFoundListener { ev -> + log.debug( + "[WebRtcEndpoint::{}] source: {}, timestamp: {}, tags: {}, candidate: {}", + ev.type, ev.source.name, ev.timestampMillis, + ev.tags, JsonUtils.toJson(ev.candidate) + ) + val message = JsonObject() + message.addProperty("id", "ADD_ICE_CANDIDATE") + message.add("candidate", JsonUtils.toJsonObject(ev.candidate)) + sendMessage(session, message.toString()) + } + + // Event: The ICE backend changed state + webRtcEp.addIceComponentStateChangedListener { ev -> + log.debug( + "[WebRtcEndpoint::{}] source: {}, timestamp: {}, tags: {}, streamId: {}, componentId: {}, state: {}", + ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.streamId, ev.componentId, ev.state + ) + } + + // Event: The ICE backend finished gathering ICE candidates + webRtcEp.addIceGatheringDoneListener { ev -> + log.info( + "[WebRtcEndpoint::{}] source: {}, timestamp: {}, tags: {}", + ev.type, ev.source.name, ev.timestampMillis, + ev.tags + ) + } + + // Event: The ICE backend selected a new pair of ICE candidates for use + webRtcEp.addNewCandidatePairSelectedListener { ev -> + log.info( + "[WebRtcEndpoint::{}] name: {}, timestamp: {}, tags: {}, streamId: {}, local: {}, remote: {}", + ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.candidatePair.streamId, + ev.candidatePair.localCandidate, + ev.candidatePair.remoteCandidate + ) + } + } + + private fun initWebRtcEndpoint( + session: WebSocketSession, + webRtcEp: WebRtcEndpoint, sdpOffer: String + ) { + initBaseEventListeners(session, webRtcEp, "WebRtcEndpoint") + initWebRtcEventListeners(session, webRtcEp) + + val sessionId = session.id + val name = "user" + sessionId + "_webrtcendpoint" + webRtcEp.name = name + + /* + OPTIONAL: Force usage of an Application-specific STUN server. + Usually this is configured globally in KMS WebRTC settings file: + /etc/kurento/modules/kurento/WebRtcEndpoint.conf.ini + + But it can also be configured per-application, as shown: + + log.info("[HelloWorldHandler::initWebRtcEndpoint] Using STUN server: 193.147.51.12:3478"); + webRtcEp.setStunServerAddress("193.147.51.12"); + webRtcEp.setStunServerPort(3478); + */ + + // Continue the SDP Negotiation: Generate an SDP Answer + val sdpAnswer = webRtcEp.processOffer(sdpOffer) + + log.info( + "[HelloWorldHandler::initWebRtcEndpoint] name: {}, SDP Offer from browser to KMS:\n{}", + name, sdpOffer + ) + log.info( + "[HelloWorldHandler::initWebRtcEndpoint] name: {}, SDP Answer from KMS to browser:\n{}", + name, sdpAnswer + ) + + val message = JsonObject() + message.addProperty("id", "PROCESS_SDP_ANSWER") + message.addProperty("sdpAnswer", sdpAnswer) + sendMessage(session, message.toString()) + } + + private fun startWebRtcEndpoint(webRtcEp: WebRtcEndpoint) { + // Calling gatherCandidates() is when the Endpoint actually starts working. + // In this tutorial, this is emphasized for demonstration purposes by + // launching the ICE candidate gathering in its own method. + webRtcEp.gatherCandidates() + } + + private fun handleProcessSdpOffer( + session: WebSocketSession, + jsonMessage: JsonObject + ) { + // ---- Session handling + + val sessionId = session.id + + log.info("[HelloWorldHandler::handleStart] User count: {}", users.size) + log.info("[HelloWorldHandler::handleStart] New user, id: {}", sessionId) + + val user = UserSession() + users[sessionId] = user + + + // ---- Media pipeline + log.info("[HelloWorldHandler::handleStart] Create Media Pipeline") + + val pipeline = kurento!!.createMediaPipeline() + user.mediaPipeline = pipeline + + val webRtcEp = + WebRtcEndpoint.Builder(pipeline).build() + user.webRtcEndpoint = webRtcEp + webRtcEp.connect(webRtcEp) + + + // ---- Endpoint configuration + val sdpOffer = jsonMessage["sdpOffer"].asString + initWebRtcEndpoint(session, webRtcEp, sdpOffer) + + log.info( + "[HelloWorldHandler::handleStart] New WebRtcEndpoint: {}", + webRtcEp.name + ) + + + // ---- Endpoint startup + startWebRtcEndpoint(webRtcEp) + + + // ---- Debug + // final String pipelineDot = pipeline.getGstreamerDot(); + // try (PrintWriter out = new PrintWriter("pipeline.dot")) { + // out.println(pipelineDot); + // } catch (IOException ex) { + // log.error("[HelloWorldHandler::start] Exception: {}", ex.getMessage()); + // } + } + + // ADD_ICE_CANDIDATE --------------------------------------------------------- + private fun handleAddIceCandidate( + session: WebSocketSession, + jsonMessage: JsonObject + ) { + val sessionId = session.id + if (!users.containsKey(sessionId)) { + log.warn( + "[HelloWorldHandler::handleAddIceCandidate] Skip, unknown user, id: {}", + sessionId + ) + return + } + + val user = users[sessionId] + val jsonCandidate = + jsonMessage["candidate"].asJsonObject + val candidate = + IceCandidate( + jsonCandidate["candidate"].asString, + jsonCandidate["sdpMid"].asString, + jsonCandidate["sdpMLineIndex"].asInt + ) + + val webRtcEp = user!!.webRtcEndpoint + webRtcEp!!.addIceCandidate(candidate) + } + + // STOP ---------------------------------------------------------------------- + private fun stop(session: WebSocketSession) { + // Remove the user session and release all resources + val user = users.remove(session.id) + if (user != null) { + val mediaPipeline = user.mediaPipeline + if (mediaPipeline != null) { + log.info("[HelloWorldHandler::stop] Release the Media Pipeline") + mediaPipeline.release() + } + } + } + + private fun handleStop( + session: WebSocketSession, + jsonMessage: JsonObject + ) { + stop(session) + } + + // ERROR --------------------------------------------------------------------- + private fun handleError( + session: WebSocketSession, + jsonMessage: JsonObject + ) { + val errMsg = jsonMessage["message"].asString + log.error("Browser error: $errMsg") + + log.info("Assume that the other side stops after an error...") + stop(session) + } // --------------------------------------------------------------------------- + + companion object { + private val log: Logger = LoggerFactory.getLogger(HelloWorldHandler::class.java) + private val gson: Gson = GsonBuilder().create() + } +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/KurentoWebsocketConfigurer.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/KurentoWebsocketConfigurer.kt new file mode 100644 index 0000000..1388e24 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/KurentoWebsocketConfigurer.kt @@ -0,0 +1,80 @@ +package io.openfuture.openmessenger.kurento + +import io.openfuture.openmessenger.assistant.gemini.GeminiService +import io.openfuture.openmessenger.kurento.groupcall.CallHandler +import io.openfuture.openmessenger.kurento.groupcall.RoomManager +import io.openfuture.openmessenger.kurento.recording.RecordingCallHandler +import io.openfuture.openmessenger.repository.MeetingNoteRepository +import io.openfuture.openmessenger.service.RecordingManagementService +import io.openfuture.openmessenger.service.SpeechToTextService +import org.kurento.client.KurentoClient +import org.springframework.beans.factory.annotation.Value +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.web.socket.config.annotation.EnableWebSocket +import org.springframework.web.socket.config.annotation.WebSocketConfigurer +import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry +import org.springframework.web.socket.server.standard.ServletServerContainerFactoryBean + +@Configuration +@EnableWebSocket +class KurentoWebsocketConfigurer( + @Value("\${kms.url}") + val kurentoUrl: String, + val recordingManagementService: RecordingManagementService, + private val speechToTextService: SpeechToTextService, + private val geminiService: GeminiService, + private val meetingNoteRepository: MeetingNoteRepository +) : WebSocketConfigurer { + @Bean + fun handler(): HelloWorldHandler { + return HelloWorldHandler() + } + + @Bean + fun kurentoClient(): KurentoClient { + return KurentoClient.create(kurentoUrl) + } + + @Bean + fun createServletServerContainerFactoryBean(): ServletServerContainerFactoryBean { + val container = ServletServerContainerFactoryBean() + container.setMaxTextMessageBufferSize(32768) + return container + } + + override fun registerWebSocketHandlers(registry: WebSocketHandlerRegistry) { + registry.addHandler(groupCallHandler(), "/groupcall").setAllowedOriginPatterns("*") + registry.addHandler(handler(), "/helloworld").setAllowedOriginPatterns("*") + registry.addHandler(callHandler(), "/call").setAllowedOrigins("*") + } + + @Bean + fun registry(): UserRegistry { + return UserRegistry() + } + + @Bean + fun recordingUserRegistry(): io.openfuture.openmessenger.kurento.recording.UserRegistry { + return io.openfuture.openmessenger.kurento.recording.UserRegistry() + } + + @Bean + fun roomManager(): RoomManager { + return RoomManager() + } + + @Bean + fun groupCallHandler(): CallHandler { + return CallHandler() + } + + @Bean + fun callHandler(): RecordingCallHandler { + return RecordingCallHandler( + kurentoClient(), recordingUserRegistry(), recordingManagementService, speechToTextService, geminiService, meetingNoteRepository + ) + } + +} + diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/UserRegistry.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/UserRegistry.kt new file mode 100644 index 0000000..1319794 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/UserRegistry.kt @@ -0,0 +1,34 @@ +package io.openfuture.openmessenger.kurento + +import io.openfuture.openmessenger.kurento.groupcall.UserSession +import org.springframework.web.socket.WebSocketSession +import java.util.concurrent.ConcurrentHashMap + +class UserRegistry { + private val usersByName = ConcurrentHashMap() + private val usersBySessionId = ConcurrentHashMap() + + fun register(user: UserSession) { + usersByName[user.name] = user + usersBySessionId[user.session.id] = user + } + + fun getByName(name: String?): UserSession? { + return usersByName[name] + } + + fun getBySession(session: WebSocketSession): UserSession? { + return usersBySessionId[session.id] + } + + fun exists(name: String?): Boolean { + return usersByName.keys.contains(name) + } + + fun removeBySession(session: WebSocketSession): UserSession { + val user = getBySession(session)!! + usersByName.remove(user.name) + usersBySessionId.remove(session.id) + return user + } +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/UserSession.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/UserSession.kt new file mode 100644 index 0000000..dabb9b1 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/UserSession.kt @@ -0,0 +1,9 @@ +package io.openfuture.openmessenger.kurento + +import org.kurento.client.MediaPipeline +import org.kurento.client.WebRtcEndpoint + +class UserSession { + var mediaPipeline: MediaPipeline? = null + var webRtcEndpoint: WebRtcEndpoint? = null +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/WsHandler.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/WsHandler.kt new file mode 100644 index 0000000..2e7ab11 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/WsHandler.kt @@ -0,0 +1,404 @@ +package io.openfuture.openmessenger.kurento + +import com.google.gson.Gson +import com.google.gson.GsonBuilder +import com.google.gson.JsonObject +import org.kurento.client.BaseRtpEndpoint +import org.kurento.client.IceCandidate +import org.kurento.client.KurentoClient +import org.kurento.client.WebRtcEndpoint +import org.kurento.jsonrpc.JsonUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.web.socket.CloseStatus +import org.springframework.web.socket.TextMessage +import org.springframework.web.socket.WebSocketSession +import org.springframework.web.socket.handler.TextWebSocketHandler +import java.io.IOException +import java.util.concurrent.ConcurrentHashMap + +class WsHandler : TextWebSocketHandler() { + private val users = ConcurrentHashMap() + + private val kurento: KurentoClient? = null + + @Throws(Exception::class) + override fun afterConnectionEstablished(session: WebSocketSession) { + log.info( + "[HelloWorldHandler::afterConnectionEstablished] New WebSocket connection, sessionId: {}", + session.id + ) + } + + @Throws(Exception::class) + override fun afterConnectionClosed( + session: WebSocketSession, + status: CloseStatus + ) { + if (!status.equalsCode(CloseStatus.NORMAL)) { + log.warn( + "[HelloWorldHandler::afterConnectionClosed] status: {}, sessionId: {}", + status, session.id + ) + } + + stop(session) + } + + @Throws(Exception::class) + override fun handleTextMessage( + session: WebSocketSession, + message: TextMessage + ) { + val sessionId = session.id + val jsonMessage = gson.fromJson( + message.payload, + JsonObject::class.java + ) + + log.info( + "[HelloWorldHandler::handleTextMessage] message: {}, sessionId: {}", + jsonMessage, sessionId + ) + + try { + val messageId = jsonMessage["id"].asString + when (messageId) { + "PROCESS_SDP_OFFER" -> // Start: Create user session and process SDP Offer + handleProcessSdpOffer(session, jsonMessage) + + "ADD_ICE_CANDIDATE" -> handleAddIceCandidate(session, jsonMessage) + "STOP" -> handleStop(session, jsonMessage) + "ERROR" -> handleError(session, jsonMessage) + else -> // Ignore the message + log.warn( + "[HelloWorldHandler::handleTextMessage] Skip, invalid message, id: {}", + messageId + ) + } + } catch (ex: Throwable) { + log.error( + "[HelloWorldHandler::handleTextMessage] Exception: {}, sessionId: {}", + ex, sessionId + ) + sendError(session, "[Kurento] Exception: " + ex.message) + } + } + + /** + * Handle an error from the underlying WebSocket message transport. + */ + @Throws(Exception::class) + override fun handleTransportError( + session: WebSocketSession, + exception: Throwable + ) { + log.error( + "[HelloWorldHandler::handleTransportError] Exception: {}, sessionId: {}", + exception, session.id + ) + + session.close(CloseStatus.SERVER_ERROR) + } + + @Synchronized + private fun sendMessage( + session: WebSocketSession, + message: String + ) { + log.debug("[HelloWorldHandler::sendMessage] {}", message) + + if (!session.isOpen) { + log.warn("[HelloWorldHandler::sendMessage] Skip, WebSocket session isn't open") + return + } + + val sessionId = session.id + if (!users.containsKey(sessionId)) { + log.warn( + "[HelloWorldHandler::sendMessage] Skip, unknown user, id: {}", + sessionId + ) + return + } + + try { + session.sendMessage(TextMessage(message)) + } catch (ex: IOException) { + log.error("[HelloWorldHandler::sendMessage] Exception: {}", ex.message) + } + } + + private fun sendError(session: WebSocketSession, errMsg: String) { + log.error(errMsg) + + if (users.containsKey(session.id)) { + val message = JsonObject() + message.addProperty("id", "ERROR") + message.addProperty("message", errMsg) + sendMessage(session, message.toString()) + } + } + + // PROCESS_SDP_OFFER --------------------------------------------------------- + private fun initBaseEventListeners( + session: WebSocketSession, + baseRtpEp: BaseRtpEndpoint, className: String + ) { + log.info( + "[HelloWorldHandler::initBaseEventListeners] name: {}, class: {}, sessionId: {}", + baseRtpEp.name, className, session.id + ) + + // Event: Some error happened + baseRtpEp.addErrorListener { ev -> + log.error( + "[{}::ErrorEvent] Error code {}: '{}', source: {}, timestamp: {}, tags: {}, description: {}", + className, ev.errorCode, ev.type, ev.source.name, + ev.timestampMillis, ev.tags, ev.description + ) + sendError(session, "[Kurento] " + ev.description) + stop(session) + } + + // Event: Media is flowing into this sink + baseRtpEp.addMediaFlowInStateChangedListener { ev -> + log.info( + "[{}::{}] source: {}, timestamp: {}, tags: {}, state: {}, padName: {}, mediaType: {}", + className, ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.state, ev.padName, ev.mediaType + ) + } + + // Event: Media is flowing out of this source + baseRtpEp.addMediaFlowOutStateChangedListener { ev -> + log.info( + "[{}::{}] source: {}, timestamp: {}, tags: {}, state: {}, padName: {}, mediaType: {}", + className, ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.state, ev.padName, ev.mediaType + ) + } + + // Event: [TODO write meaning of this event] + baseRtpEp.addConnectionStateChangedListener { ev -> + log.info( + "[{}::{}] source: {}, timestamp: {}, tags: {}, oldState: {}, newState: {}", + className, ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.oldState, ev.newState + ) + } + + // Event: [TODO write meaning of this event] + baseRtpEp.addMediaStateChangedListener { ev -> + log.info( + "[{}::{}] source: {}, timestamp: {}, tags: {}, oldState: {}, newState: {}", + className, ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.oldState, ev.newState + ) + } + + // Event: This element will (or will not) perform media transcoding + baseRtpEp.addMediaTranscodingStateChangedListener { ev -> + log.info( + "[{}::{}] source: {}, timestamp: {}, tags: {}, state: {}, binName: {}, mediaType: {}", + className, ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.state, ev.binName, ev.mediaType + ) + } + } + + private fun initWebRtcEventListeners( + session: WebSocketSession, + webRtcEp: WebRtcEndpoint + ) { + log.info( + "[HelloWorldHandler::initWebRtcEventListeners] name: {}, sessionId: {}", + webRtcEp.name, session.id + ) + + // Event: The ICE backend found a local candidate during Trickle ICE + webRtcEp.addIceCandidateFoundListener { ev -> + log.debug( + "[WebRtcEndpoint::{}] source: {}, timestamp: {}, tags: {}, candidate: {}", + ev.type, ev.source.name, ev.timestampMillis, + ev.tags, JsonUtils.toJson(ev.candidate) + ) + val message = JsonObject() + message.addProperty("id", "ADD_ICE_CANDIDATE") + message.add("candidate", JsonUtils.toJsonObject(ev.candidate)) + sendMessage(session, message.toString()) + } + + // Event: The ICE backend changed state + webRtcEp.addIceComponentStateChangedListener { ev -> + log.debug( + "[WebRtcEndpoint::{}] source: {}, timestamp: {}, tags: {}, streamId: {}, componentId: {}, state: {}", + ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.streamId, ev.componentId, ev.state + ) + } + + // Event: The ICE backend finished gathering ICE candidates + webRtcEp.addIceGatheringDoneListener { ev -> + log.info( + "[WebRtcEndpoint::{}] source: {}, timestamp: {}, tags: {}", + ev.type, ev.source.name, ev.timestampMillis, + ev.tags + ) + } + + // Event: The ICE backend selected a new pair of ICE candidates for use + webRtcEp.addNewCandidatePairSelectedListener { ev -> + log.info( + "[WebRtcEndpoint::{}] name: {}, timestamp: {}, tags: {}, streamId: {}, local: {}, remote: {}", + ev.type, ev.source.name, ev.timestampMillis, + ev.tags, ev.candidatePair.streamId, + ev.candidatePair.localCandidate, + ev.candidatePair.remoteCandidate + ) + } + } + + private fun initWebRtcEndpoint( + session: WebSocketSession, + webRtcEp: WebRtcEndpoint, sdpOffer: String + ) { + initBaseEventListeners(session, webRtcEp, "WebRtcEndpoint") + initWebRtcEventListeners(session, webRtcEp) + + val sessionId = session.id + val name = "user" + sessionId + "_webrtcendpoint" + webRtcEp.name = name + + /* + OPTIONAL: Force usage of an Application-specific STUN server. + Usually this is configured globally in KMS WebRTC settings file: + /etc/kurento/modules/kurento/WebRtcEndpoint.conf.ini + + But it can also be configured per-application, as shown: + + log.info("[HelloWorldHandler::initWebRtcEndpoint] Using STUN server: 193.147.51.12:3478"); + webRtcEp.setStunServerAddress("193.147.51.12"); + webRtcEp.setStunServerPort(3478); + */ + + // Continue the SDP Negotiation: Generate an SDP Answer + val sdpAnswer = webRtcEp.processOffer(sdpOffer) + + log.info( + "[HelloWorldHandler::initWebRtcEndpoint] name: {}, SDP Offer from browser to KMS:\n{}", + name, sdpOffer + ) + log.info( + "[HelloWorldHandler::initWebRtcEndpoint] name: {}, SDP Answer from KMS to browser:\n{}", + name, sdpAnswer + ) + + val message = JsonObject() + message.addProperty("id", "PROCESS_SDP_ANSWER") + message.addProperty("sdpAnswer", sdpAnswer) + sendMessage(session, message.toString()) + } + + private fun startWebRtcEndpoint(webRtcEp: WebRtcEndpoint) { + webRtcEp.gatherCandidates() + } + + private fun handleProcessSdpOffer( + session: WebSocketSession, + jsonMessage: JsonObject + ) { + // ---- Session handling + + val sessionId = session.id + + log.info("[HelloWorldHandler::handleStart] User count: {}", users.size) + log.info("[HelloWorldHandler::handleStart] New user, id: {}", sessionId) + + val user = UserSession() + users[sessionId] = user + + log.info("[HelloWorldHandler::handleStart] Create Media Pipeline") + + val pipeline = kurento!!.createMediaPipeline() + user.mediaPipeline = pipeline + + val webRtcEp = + WebRtcEndpoint.Builder(pipeline).build() + user.webRtcEndpoint = webRtcEp + webRtcEp.connect(webRtcEp) + + val sdpOffer = jsonMessage["sdpOffer"].asString + initWebRtcEndpoint(session, webRtcEp, sdpOffer) + + log.info( + "[HelloWorldHandler::handleStart] New WebRtcEndpoint: {}", + webRtcEp.name + ) + + startWebRtcEndpoint(webRtcEp) + } + + private fun handleAddIceCandidate( + session: WebSocketSession, + jsonMessage: JsonObject + ) { + val sessionId = session.id + if (!users.containsKey(sessionId)) { + log.warn( + "[HelloWorldHandler::handleAddIceCandidate] Skip, unknown user, id: {}", + sessionId + ) + return + } + + val user = users[sessionId] + val jsonCandidate = + jsonMessage["candidate"].asJsonObject + val candidate = + IceCandidate( + jsonCandidate["candidate"].asString, + jsonCandidate["sdpMid"].asString, + jsonCandidate["sdpMLineIndex"].asInt + ) + + val webRtcEp = user?.webRtcEndpoint + webRtcEp!!.addIceCandidate(candidate) + } + + // STOP ---------------------------------------------------------------------- + private fun stop(session: WebSocketSession) { + // Remove the user session and release all resources + val user = users.remove(session.id) + if (user != null) { + val mediaPipeline = user.mediaPipeline + if (mediaPipeline != null) { + log.info("[HelloWorldHandler::stop] Release the Media Pipeline") + mediaPipeline.release() + } + } + } + + private fun handleStop( + session: WebSocketSession, + jsonMessage: JsonObject + ) { + stop(session) + } + + private fun handleError( + session: WebSocketSession, + jsonMessage: JsonObject + ) { + val errMsg = jsonMessage["message"].asString + log.error("Browser error: $errMsg") + + log.info("Assume that the other side stops after an error...") + stop(session) + } + + companion object { + private val log: Logger = LoggerFactory.getLogger(WsHandler::class.java) + private val gson: Gson = GsonBuilder().create() + } +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/groupcall/CallHandler.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/groupcall/CallHandler.kt new file mode 100644 index 0000000..15e5e8b --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/groupcall/CallHandler.kt @@ -0,0 +1,93 @@ +package io.openfuture.openmessenger.kurento.groupcall + +import com.google.gson.Gson +import com.google.gson.GsonBuilder +import com.google.gson.JsonObject +import io.openfuture.openmessenger.kurento.UserRegistry +import org.kurento.client.IceCandidate +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.web.socket.CloseStatus +import org.springframework.web.socket.TextMessage +import org.springframework.web.socket.WebSocketSession +import org.springframework.web.socket.handler.TextWebSocketHandler +import java.io.IOException + +class CallHandler : TextWebSocketHandler() { + @Autowired + private val roomManager: RoomManager? = null + + @Autowired + private val registry: UserRegistry? = null + + @Throws(Exception::class) + public override fun handleTextMessage(session: WebSocketSession, message: TextMessage) { + val jsonMessage = gson.fromJson(message.payload, JsonObject::class.java) + + val user = registry!!.getBySession(session) + + if (user != null) { + log.debug("Incoming message from user '{}': {}", user.name, jsonMessage) + } else { + log.debug("Incoming message from new user: {}", jsonMessage) + } + + when (jsonMessage["id"].asString) { + "joinRoom" -> joinRoom(jsonMessage, session) + "receiveVideoFrom" -> { + val senderName = jsonMessage["sender"].asString + val sender = registry.getByName(senderName) + val sdpOffer = jsonMessage["sdpOffer"].asString + user!!.receiveVideoFrom(sender!!, sdpOffer) + } + + "leaveRoom" -> leaveRoom(user!!) + "onIceCandidate" -> { + val candidate = jsonMessage["candidate"].asJsonObject + + if (user != null) { + val cand = IceCandidate( + candidate["candidate"].asString, + candidate["sdpMid"].asString, candidate["sdpMLineIndex"].asInt + ) + user.addCandidate(cand, jsonMessage["name"].asString) + } + } + + else -> {} + } + } + + @Throws(Exception::class) + override fun afterConnectionClosed(session: WebSocketSession, status: CloseStatus) { + val user = registry!!.removeBySession(session) + roomManager!!.getRoom(user.roomName).leave(user) + } + + @Throws(IOException::class) + private fun joinRoom(params: JsonObject, session: WebSocketSession) { + val roomName = params["room"].asString + val name = params["name"].asString + log.info("PARTICIPANT {}: trying to join room {}", name, roomName) + + val room = roomManager!!.getRoom(roomName) + val user = room.join(name, session) + registry!!.register(user) + } + + @Throws(IOException::class) + private fun leaveRoom(user: UserSession) { + val room = roomManager!!.getRoom(user.roomName) + room.leave(user) + if (room.participants.isEmpty()) { + roomManager.removeRoom(room) + } + } + + companion object { + private val log: Logger = LoggerFactory.getLogger(CallHandler::class.java) + + private val gson: Gson = GsonBuilder().create() + } +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/groupcall/Room.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/groupcall/Room.kt new file mode 100644 index 0000000..1ca4a67 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/groupcall/Room.kt @@ -0,0 +1,152 @@ +package io.openfuture.openmessenger.kurento.groupcall + +import com.google.gson.* +import org.kurento.client.* +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.web.socket.WebSocketSession +import java.io.Closeable +import java.io.IOException +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentMap +import javax.annotation.PreDestroy + +class Room(val name: String?, private val pipeline: MediaPipeline) : Closeable { + private val log: Logger = LoggerFactory.getLogger(Room::class.java) + + val participants: ConcurrentMap = ConcurrentHashMap() + + init { + log.info("ROOM {} has been created", name) + } + + @PreDestroy + private fun shutdown() { + this.close() + } + + @Throws(IOException::class) + fun join(userName: String, session: WebSocketSession): UserSession { + log.info("ROOM {}: adding participant {}", this.name, userName) + val participant = UserSession(userName, this.name, session, this.pipeline) + joinRoom(participant) + participants[participant.name] = participant + sendParticipantNames(participant) + return participant + } + + @Throws(IOException::class) + fun leave(user: UserSession) { + log.debug("PARTICIPANT {}: Leaving room {}", user.name, this.name) + this.removeParticipant(user.name) + user.close() + } + + @Throws(IOException::class) + private fun joinRoom(newParticipant: UserSession): Collection { + val newParticipantMsg = JsonObject() + newParticipantMsg.addProperty("id", "newParticipantArrived") + newParticipantMsg.addProperty("name", newParticipant.name) + + val participantsList: MutableList = ArrayList(participants.values.size) + log.debug( + "ROOM {}: notifying other participants of new participant {}", name, + newParticipant.name + ) + + for (participant in participants.values) { + try { + participant.sendMessage(newParticipantMsg) + } catch (e: IOException) { + log.debug("ROOM {}: participant {} could not be notified", name, participant.name, e) + } + participantsList.add(participant.name) + } + + return participantsList + } + + @Throws(IOException::class) + private fun removeParticipant(name: String?) { + participants.remove(name) + + log.debug("ROOM {}: notifying all users that {} is leaving the room", this.name, name) + + val unnotifiedParticipants: MutableList = ArrayList() + val participantLeftJson = JsonObject() + participantLeftJson.addProperty("id", "participantLeft") + participantLeftJson.addProperty("name", name) + for (participant in participants.values) { + try { + participant.cancelVideoFrom(name) + participant.sendMessage(participantLeftJson) + } catch (e: IOException) { + unnotifiedParticipants.add(participant.name) + } + } + + if (!unnotifiedParticipants.isEmpty()) { + log.debug( + "ROOM {}: The users {} could not be notified that {} left the room", this.name, + unnotifiedParticipants, name + ) + } + } + + @Throws(IOException::class) + fun sendParticipantNames(user: UserSession) { + val participantsArray = JsonArray() + for (participant in this.getParticipants()) { + if (participant != user) { + val participantName: JsonElement = JsonPrimitive(participant.name) + participantsArray.add(participantName) + } + } + + val existingParticipantsMsg = JsonObject() + existingParticipantsMsg.addProperty("id", "existingParticipants") + existingParticipantsMsg.add("data", participantsArray) + log.debug( + "PARTICIPANT {}: sending a list of {} participants", user.name, + participantsArray.size() + ) + user.sendMessage(existingParticipantsMsg) + } + + fun getParticipants(): Collection { + return participants.values + } + + fun getParticipant(name: String?): UserSession? { + return participants[name] + } + + override fun close() { + for (user in participants.values) { + try { + user.close() + } catch (e: IOException) { + log.debug( + "ROOM {}: Could not invoke close on participant {}", this.name, user.name, + e + ) + } + } + + participants.clear() + + pipeline.release(object : Continuation { + @Throws(Exception::class) + override fun onSuccess(result: Void?) { + log.trace("ROOM {}: Released Pipeline", this@Room.name) + } + + @Throws(Exception::class) + override fun onError(cause: Throwable) { + log.warn("PARTICIPANT {}: Could not release Pipeline", this@Room.name) + } + }) + + log.debug("Room {} closed", this.name) + } +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/groupcall/RoomManager.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/groupcall/RoomManager.kt new file mode 100644 index 0000000..0491976 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/groupcall/RoomManager.kt @@ -0,0 +1,52 @@ +/* + * (C) Copyright 2014 Kurento (http://kurento.org/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package io.openfuture.openmessenger.kurento.groupcall + +import org.kurento.client.KurentoClient +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentMap + +class RoomManager { + private val log: Logger = LoggerFactory.getLogger(RoomManager::class.java) + + @Autowired + private val kurento: KurentoClient? = null + + private val rooms: ConcurrentMap = ConcurrentHashMap() + + fun getRoom(roomName: String?): Room { + log.debug("Searching for room {}", roomName) + var room = rooms[roomName] + + if (room == null) { + log.debug("Room {} not existent. Will create now!", roomName) + room = Room(roomName, kurento!!.createMediaPipeline()) + rooms[roomName] = room + } + log.debug("Room {} found!", roomName) + return room + } + + fun removeRoom(room: Room) { + rooms.remove(room.name) + room.close() + log.info("Room {} removed and closed", room.name) + } +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/groupcall/UserSession.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/groupcall/UserSession.kt new file mode 100644 index 0000000..03a8cec --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/groupcall/UserSession.kt @@ -0,0 +1,263 @@ +package io.openfuture.openmessenger.kurento.groupcall + +import com.google.gson.JsonObject +import org.kurento.client.* +import org.kurento.jsonrpc.JsonUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.web.socket.TextMessage +import org.springframework.web.socket.WebSocketSession +import java.io.Closeable +import java.io.IOException +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentMap + +class UserSession( + val name: String, + + val roomName: String?, val session: WebSocketSession, + private val pipeline: MediaPipeline +) : Closeable { + val outgoingWebRtcPeer: WebRtcEndpoint = WebRtcEndpoint.Builder(pipeline).build() + private val incomingMedia: ConcurrentMap = ConcurrentHashMap() + + init { + outgoingWebRtcPeer.addIceCandidateFoundListener { event -> + val response = JsonObject() + response.addProperty("id", "iceCandidate") + response.addProperty("name", name) + response.add("candidate", JsonUtils.toJsonObject(event.candidate)) + try { + synchronized(session) { + session.sendMessage(TextMessage(response.toString())) + } + } catch (e: IOException) { + log.debug(e.message) + } + } + } + + @Throws(IOException::class) + fun receiveVideoFrom(sender: UserSession, sdpOffer: String?) { + log.info( + "USER {}: connecting with {} in room {}", + this.name, + sender.name, + this.roomName + ) + + log.trace( + "USER {}: SdpOffer for {} is {}", + this.name, + sender.name, sdpOffer + ) + + val ipSdpAnswer = getEndpointForUser(sender)!!.processOffer(sdpOffer) + val scParams = JsonObject() + scParams.addProperty("id", "receiveVideoAnswer") + scParams.addProperty("name", sender.name) + scParams.addProperty("sdpAnswer", ipSdpAnswer) + + log.trace( + "USER {}: SdpAnswer for {} is {}", + this.name, + sender.name, ipSdpAnswer + ) + this.sendMessage(scParams) + log.debug("gather candidates") + getEndpointForUser(sender)!!.gatherCandidates() + } + + private fun getEndpointForUser(sender: UserSession): WebRtcEndpoint? { + if (sender.name == name) { + log.debug( + "PARTICIPANT {}: configuring loopback", + this.name + ) + return outgoingWebRtcPeer + } + + log.debug( + "PARTICIPANT {}: receiving video from {}", + this.name, + sender.name + ) + + var incoming = incomingMedia[sender.name] + if (incoming == null) { + log.debug( + "PARTICIPANT {}: creating new endpoint for {}", + this.name, + sender.name + ) + incoming = WebRtcEndpoint.Builder(pipeline).build() + + incoming.addIceCandidateFoundListener(EventListener { event -> + val response = JsonObject() + response.addProperty("id", "iceCandidate") + response.addProperty("name", sender.name) + response.add("candidate", JsonUtils.toJsonObject(event.candidate)) + try { + synchronized(session) { + session.sendMessage(TextMessage(response.toString())) + } + } catch (e: IOException) { + log.debug(e.message) + } + }) + + incomingMedia[sender.name] = incoming + } + + log.debug( + "PARTICIPANT {}: obtained endpoint for {}", + this.name, + sender.name + ) + sender.outgoingWebRtcPeer.connect(incoming) + + return incoming + } + + fun cancelVideoFrom(sender: UserSession) { + this.cancelVideoFrom(sender.name) + } + + fun cancelVideoFrom(senderName: String?) { + log.debug( + "PARTICIPANT {}: canceling video reception from {}", + this.name, senderName + ) + val incoming = incomingMedia.remove(senderName) + + log.debug( + "PARTICIPANT {}: removing endpoint for {}", + this.name, senderName + ) + incoming!!.release(object : Continuation { + @Throws(Exception::class) + override fun onSuccess(result: Void?) { + log.trace( + "PARTICIPANT {}: Released successfully incoming EP for {}", + this@UserSession.name, senderName + ) + } + + @Throws(Exception::class) + override fun onError(cause: Throwable) { + log.warn( + "PARTICIPANT {}: Could not release incoming EP for {}", + this@UserSession.name, + senderName + ) + } + }) + } + + @Throws(IOException::class) + override fun close() { + log.debug( + "PARTICIPANT {}: Releasing resources", + this.name + ) + for (remoteParticipantName in incomingMedia.keys) { + log.trace( + "PARTICIPANT {}: Released incoming EP for {}", + this.name, remoteParticipantName + ) + + val ep = incomingMedia[remoteParticipantName] + + ep!!.release(object : Continuation { + @Throws(Exception::class) + override fun onSuccess(result: Void?) { + log.trace( + "PARTICIPANT {}: Released successfully incoming EP for {}", + this@UserSession.name, remoteParticipantName + ) + } + + @Throws(Exception::class) + override fun onError(cause: Throwable) { + log.warn( + "PARTICIPANT {}: Could not release incoming EP for {}", + this@UserSession.name, + remoteParticipantName + ) + } + }) + } + + outgoingWebRtcPeer.release(object : Continuation { + @Throws(Exception::class) + override fun onSuccess(result: Void?) { + log.trace( + "PARTICIPANT {}: Released outgoing EP", + this@UserSession.name + ) + } + + @Throws(Exception::class) + override fun onError(cause: Throwable) { + log.warn( + "USER {}: Could not release outgoing EP", + this@UserSession.name + ) + } + }) + } + + @Throws(IOException::class) + fun sendMessage(message: JsonObject) { + log.debug( + "USER {}: Sending message {}", + name, message + ) + synchronized(session) { + session.sendMessage(TextMessage(message.toString())) + } + } + + fun addCandidate(candidate: IceCandidate?, name: String) { + if (this.name.compareTo(name) == 0) { + outgoingWebRtcPeer.addIceCandidate(candidate) + } else { + val webRtc = incomingMedia[name] + webRtc?.addIceCandidate(candidate) + } + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + override fun equals(obj: Any?): Boolean { + if (this === obj) { + return true + } + if (obj == null || obj !is UserSession) { + return false + } + val other = obj + var eq = name == other.name + eq = eq and (roomName == other.roomName) + return eq + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#hashCode() + */ + override fun hashCode(): Int { + var result = 1 + result = 31 * result + name.hashCode() + result = 31 * result + roomName.hashCode() + return result + } + + companion object { + private val log: Logger = LoggerFactory.getLogger(UserSession::class.java) + } +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/CallMediaPipeline.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/CallMediaPipeline.kt new file mode 100644 index 0000000..627b020 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/CallMediaPipeline.kt @@ -0,0 +1,64 @@ +package io.openfuture.openmessenger.kurento.recording + +import org.kurento.client.* +import java.text.SimpleDateFormat +import java.util.* + + +class CallMediaPipeline(kurento: KurentoClient, from: String, to: String?) { + val pipeline: MediaPipeline = kurento.createMediaPipeline() + + val callerWebRtcEp: WebRtcEndpoint = WebRtcEndpoint.Builder(pipeline).build() + val calleeWebRtcEp: WebRtcEndpoint = WebRtcEndpoint.Builder(pipeline).build() + private val recorderCaller: RecorderEndpoint + private val recorderCallee: RecorderEndpoint + + val composite: Composite = Composite.Builder(pipeline).build() + val out = HubPort.Builder(composite).build() + val recordedFileUri = RECORDING_PATH + "combined" + RECORDING_EXT + val recorder: RecorderEndpoint = RecorderEndpoint.Builder(pipeline, recordedFileUri) + .build() + + init { + recorderCaller = RecorderEndpoint.Builder(pipeline, RECORDING_PATH + from + RECORDING_EXT) + .build() + recorderCallee = RecorderEndpoint.Builder(pipeline, RECORDING_PATH + to + RECORDING_EXT) + .build() + + callerWebRtcEp.connect(calleeWebRtcEp) + callerWebRtcEp.connect(recorderCaller) + + calleeWebRtcEp.connect(callerWebRtcEp) + calleeWebRtcEp.connect(recorderCallee) + + // mixing + val callerPort: HubPort = HubPort.Builder(composite).build() + val calleePort: HubPort = HubPort.Builder(composite).build() + + + callerWebRtcEp.connect(callerPort) + calleeWebRtcEp.connect(calleePort) + + out.connect(recorder) + } + + fun record() { + recorderCaller.record() + recorderCallee.record() + recorder.record() + } + + fun generateSdpAnswerForCaller(sdpOffer: String?): String { + return callerWebRtcEp.processOffer(sdpOffer) + } + + fun generateSdpAnswerForCallee(sdpOffer: String?): String { + return calleeWebRtcEp.processOffer(sdpOffer) + } + + companion object { + private val df = SimpleDateFormat("yyyy-MM-dd_HH-mm-ss-S") + val RECORDING_PATH: String = "file:///tmp/recordings/" + df.format(Date()) + "-" + const val RECORDING_EXT: String = ".webm" + } +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/PlayMediaPipeline.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/PlayMediaPipeline.kt new file mode 100644 index 0000000..a786af6 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/PlayMediaPipeline.kt @@ -0,0 +1,59 @@ +package io.openfuture.openmessenger.kurento.recording + +import com.google.gson.JsonObject +import io.openfuture.openmessenger.kurento.recording.CallMediaPipeline.Companion.RECORDING_EXT +import io.openfuture.openmessenger.kurento.recording.CallMediaPipeline.Companion.RECORDING_PATH +import org.kurento.client.KurentoClient +import org.kurento.client.MediaPipeline +import org.kurento.client.PlayerEndpoint +import org.kurento.client.WebRtcEndpoint +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.web.socket.TextMessage +import org.springframework.web.socket.WebSocketSession +import java.io.IOException + +class PlayMediaPipeline(kurento: KurentoClient, user: String, session: WebSocketSession) { + val pipeline: MediaPipeline = kurento.createMediaPipeline() + var webRtc: WebRtcEndpoint? + val player: PlayerEndpoint + + init { + webRtc = WebRtcEndpoint.Builder(pipeline).build() + player = PlayerEndpoint.Builder(pipeline, RECORDING_PATH + user + RECORDING_EXT).build() + + player.connect(webRtc) + + player.addErrorListener { event -> + log.info("ErrorEvent: {}", event.description) + sendPlayEnd(session) + } + } + + fun sendPlayEnd(session: WebSocketSession) { + try { + val response: JsonObject = JsonObject() + response.addProperty("id", "playEnd") + session.sendMessage(TextMessage(response.toString())) + } catch (e: IOException) { + log.error("Error sending playEndOfStream message", e) + } + + // Release pipeline + pipeline.release() + this.webRtc = null + } + + fun play() { + player.play() + } + + fun generateSdpAnswer(sdpOffer: String?): String { + val processOffer: String? = webRtc?.processOffer(sdpOffer) + return processOffer!! + } + + companion object { + private val log: Logger = LoggerFactory.getLogger(PlayMediaPipeline::class.java) + } +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/RecordingCallHandler.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/RecordingCallHandler.kt new file mode 100644 index 0000000..67626fe --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/RecordingCallHandler.kt @@ -0,0 +1,345 @@ +package io.openfuture.openmessenger.kurento.recording + +import com.google.gson.Gson +import com.google.gson.GsonBuilder +import com.google.gson.JsonObject +import io.openfuture.openmessenger.assistant.gemini.GeminiService +import io.openfuture.openmessenger.repository.MeetingNoteRepository +import io.openfuture.openmessenger.repository.entity.MeetingNoteEntity +import io.openfuture.openmessenger.service.RecordingManagementService +import io.openfuture.openmessenger.service.SpeechToTextService +import org.kurento.client.IceCandidate +import org.kurento.client.KurentoClient +import org.kurento.client.MediaPipeline +import org.kurento.jsonrpc.JsonUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.web.socket.CloseStatus +import org.springframework.web.socket.TextMessage +import org.springframework.web.socket.WebSocketSession +import org.springframework.web.socket.handler.TextWebSocketHandler +import java.io.IOException +import java.time.LocalDateTime +import java.util.concurrent.ConcurrentHashMap + +class RecordingCallHandler( + private val kurento: KurentoClient, + private val registry: UserRegistry, + private val recordingManagementService: RecordingManagementService, + private val speechToTextService: SpeechToTextService, + private val geminiService: GeminiService, + private val meetingNoteRepository: MeetingNoteRepository +): TextWebSocketHandler() { + val pipelines = ConcurrentHashMap() + val recordings = ConcurrentHashMap() + + @Throws(Exception::class) + public override fun handleTextMessage(session: WebSocketSession, message: TextMessage) { + val jsonMessage = gson.fromJson( + message.payload, + JsonObject::class.java + ) + val user = registry.getBySession(session) + + if (user != null) { + log.debug("Incoming message from user '{}': {}", user.name, jsonMessage) + } else { + log.debug("Incoming message from new user: {}", jsonMessage) + } + + when (jsonMessage["id"].asString) { + "register" -> register(session, jsonMessage) + "call" -> call(user!!, jsonMessage) + "incomingCallResponse" -> incomingCallResponse(user!!, jsonMessage) + "play" -> play(user!!, jsonMessage) + "onIceCandidate" -> { + val candidate = jsonMessage["candidate"].asJsonObject + + if (user != null) { + val cand = + IceCandidate( + candidate["candidate"].asString, candidate["sdpMid"] + .asString, candidate["sdpMLineIndex"].asInt + ) + user.addCandidate(cand) + } + } + + "stop" -> { + stop(session) + releasePipeline(user!!) + } + + "stopPlay" -> releasePipeline(user!!) + else -> {} + } + } + + @Throws(IOException::class) + private fun register(session: WebSocketSession, jsonMessage: JsonObject) { + val name = jsonMessage.getAsJsonPrimitive("name").asString + + val caller = UserSession(session, name) + var responseMsg = "accepted" + if (name.isEmpty()) { + responseMsg = "rejected: empty user name" + } else if (registry!!.exists(name)) { + responseMsg = "rejected: user '$name' already registered" + } else { + registry.register(caller) + } + + val response = JsonObject() + response.addProperty("id", "registerResponse") + response.addProperty("response", responseMsg) + caller.sendMessage(response) + } + + @Throws(IOException::class) + private fun call(caller: UserSession, jsonMessage: JsonObject) { + val to = jsonMessage["to"].asString + val from = jsonMessage["from"].asString + val response = JsonObject() + + if (registry!!.exists(to)) { + caller.sdpOffer = jsonMessage.getAsJsonPrimitive("sdpOffer").asString + caller.callingTo = to + + response.addProperty("id", "incomingCall") + response.addProperty("from", from) + + val callee = registry.getByName(to) + callee!!.sendMessage(response) + callee.callingFrom = from + } else { + response.addProperty("id", "callResponse") + response.addProperty("response", "rejected") + response.addProperty("message", "user '$to' is not registered") + + caller.sendMessage(response) + } + } + + @Throws(IOException::class) + private fun incomingCallResponse(callee: UserSession, jsonMessage: JsonObject) { + val callResponse = jsonMessage["callResponse"].asString + val from = jsonMessage["from"].asString + val caller = registry.getByName(from) + val to = caller?.callingTo + + if ("accept" == callResponse) { + log.debug("Accepted call from '{}' to '{}'", from, to) + + val callMediaPipeline = CallMediaPipeline(kurento, from, to) + recordings[caller!!.sessionId] = callMediaPipeline.recordedFileUri + recordings[callee.sessionId] = callMediaPipeline.recordedFileUri + pipelines[caller.sessionId] = callMediaPipeline.pipeline + pipelines[callee.sessionId] = callMediaPipeline.pipeline + + callee.setWebRtcEndpoint(callMediaPipeline.calleeWebRtcEp) + callMediaPipeline.calleeWebRtcEp.addIceCandidateFoundListener { event -> + val response = JsonObject() + response.addProperty("id", "iceCandidate") + response.add("candidate", JsonUtils.toJsonObject(event.candidate)) + try { + synchronized(callee.session) { + callee.session.sendMessage(TextMessage(response.toString())) + } + } catch (e: IOException) { + log.debug(e.message) + } + } + + val calleeSdpOffer = jsonMessage["sdpOffer"].asString + val calleeSdpAnswer = callMediaPipeline.generateSdpAnswerForCallee(calleeSdpOffer) + val startCommunication = JsonObject() + startCommunication.addProperty("id", "startCommunication") + startCommunication.addProperty("sdpAnswer", calleeSdpAnswer) + + synchronized(callee) { + callee.sendMessage(startCommunication) + } + + callMediaPipeline.calleeWebRtcEp.gatherCandidates() + + val callerSdpOffer = registry.getByName(from)?.sdpOffer + + caller.setWebRtcEndpoint(callMediaPipeline.callerWebRtcEp) + callMediaPipeline.callerWebRtcEp.addIceCandidateFoundListener { event -> + val response = JsonObject() + response.addProperty("id", "iceCandidate") + response.add("candidate", JsonUtils.toJsonObject(event.candidate)) + try { + synchronized(caller.session) { + caller.session.sendMessage(TextMessage(response.toString())) + } + } catch (e: IOException) { + log.debug(e.message) + } + } + + val callerSdpAnswer = callMediaPipeline.generateSdpAnswerForCaller(callerSdpOffer) + + val response = JsonObject() + response.addProperty("id", "callResponse") + response.addProperty("response", "accepted") + response.addProperty("sdpAnswer", callerSdpAnswer) + + synchronized(caller) { + caller.sendMessage(response) + } + + callMediaPipeline.callerWebRtcEp.gatherCandidates() + + callMediaPipeline.record() + } else { + val response = JsonObject() + response.addProperty("id", "callResponse") + response.addProperty("response", "rejected") + caller!!.sendMessage(response) + } + } + + @Throws(IOException::class) + fun stop(session: WebSocketSession) { + val stopperUser = registry.getBySession(session) + + if (stopperUser != null) { + val stoppedUser = + if ((stopperUser.callingFrom != null)) + registry.getByName(stopperUser.callingFrom) + else + if (stopperUser.callingTo != null) + registry.getByName(stopperUser.callingTo) + else + null + + if (stoppedUser != null) { + val message = JsonObject() + message.addProperty("id", "stopCommunication") + stoppedUser.sendMessage(message) + stoppedUser.clear() + } + stopperUser.clear() + + val caller = stopperUser.callingFrom ?: stoppedUser?.callingFrom + val callee = stopperUser.callingTo ?: stoppedUser?.callingTo + log.info("Caller = $caller, callee = $callee") + + pipelines[stoppedUser?.sessionId] + + val fileUriString = recordings[stoppedUser?.sessionId] + fileUriString?.let { val uploadToS3 = recordingManagementService.uploadToS3(it) + if (uploadToS3 == -1) { + log.warn("No file was uploaded for call session ${stoppedUser?.sessionId}") + return + } + val transcript = speechToTextService.extractTranscript(uploadToS3) + val chat = geminiService.chat("Generate a summary from the following meeting record: {$transcript}") + + val meetingNoteEntity = MeetingNoteEntity( + caller, + 777, + 999, + "[$caller, $callee]", + callee, + LocalDateTime.now(), + 1, + LocalDateTime.now(), + LocalDateTime.now(), + chat + ) + meetingNoteRepository.save(meetingNoteEntity) + } + } + } + + fun releasePipeline(session: UserSession) { + val sessionId = session.sessionId + + if (pipelines.containsKey(sessionId)) { + pipelines[sessionId]!!.release() + pipelines.remove(sessionId) + } + session.setWebRtcEndpoint(null) + session.playingWebRtcEndpoint = null + + // set to null the endpoint of the other user + val stoppedUser = + if ((session.callingFrom != null)) + registry!!.getByName(session.callingFrom) + else + registry!!.getByName(session.callingTo) + stoppedUser!!.setWebRtcEndpoint(null) + stoppedUser.playingWebRtcEndpoint = null + } + + @Throws(IOException::class) + private fun play(session: UserSession, jsonMessage: JsonObject) { + val user = jsonMessage["user"].asString + log.debug("Playing recorded call of user '{}'", user) + + val response = JsonObject() + response.addProperty("id", "playResponse") + + if (registry!!.getByName(user) != null && registry.getBySession(session.session) != null) { + val playMediaPipeline = + PlayMediaPipeline(kurento!!, user, session.session) + + session.playingWebRtcEndpoint = playMediaPipeline.webRtc + + playMediaPipeline.player.addEndOfStreamListener { + val user = registry.getBySession(session.session) + releasePipeline(user!!) + playMediaPipeline.sendPlayEnd(session.session) + } + + playMediaPipeline.webRtc?.addIceCandidateFoundListener { event -> + val response = JsonObject() + response.addProperty("id", "iceCandidate") + response.add("candidate", JsonUtils.toJsonObject(event.candidate)) + try { + synchronized(session) { + session.session.sendMessage(TextMessage(response.toString())) + } + } catch (e: IOException) { + log.debug(e.message) + } + } + + val sdpOffer = jsonMessage["sdpOffer"].asString + val sdpAnswer = playMediaPipeline.generateSdpAnswer(sdpOffer) + + response.addProperty("response", "accepted") + + response.addProperty("sdpAnswer", sdpAnswer) + + playMediaPipeline.play() + pipelines[session.sessionId] = playMediaPipeline.pipeline + synchronized(session.session) { + session.sendMessage(response) + } + + playMediaPipeline.webRtc?.gatherCandidates() + } else { + response.addProperty("response", "rejected") + response.addProperty( + "error", ("No recording for user '" + user + + "'. Please type a correct user in the 'Peer' field.") + ) + session.session.sendMessage(TextMessage(response.toString())) + } + } + + @Throws(Exception::class) + override fun afterConnectionClosed(session: WebSocketSession, status: CloseStatus) { + stop(session) + registry.removeBySession(session) + } + + companion object { + private val log: Logger = LoggerFactory.getLogger(RecordingCallHandler::class.java) + private val gson: Gson = GsonBuilder().create() + } + +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/UserRegistry.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/UserRegistry.kt new file mode 100644 index 0000000..a4c6244 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/UserRegistry.kt @@ -0,0 +1,35 @@ +package io.openfuture.openmessenger.kurento.recording + +import org.springframework.web.socket.WebSocketSession +import java.util.concurrent.ConcurrentHashMap + +class UserRegistry { + private val usersByName = ConcurrentHashMap() + private val usersBySessionId = ConcurrentHashMap() + + fun register(user: UserSession) { + usersByName[user.name] = user + usersBySessionId[user.session.id] = user + } + + fun getByName(name: String?): UserSession? { + return usersByName[name] + } + + fun getBySession(session: WebSocketSession): UserSession? { + return usersBySessionId[session.id] + } + + fun exists(name: String?): Boolean { + return usersByName.keys.contains(name) + } + + fun removeBySession(session: WebSocketSession): UserSession? { + val user = getBySession(session) + if (user != null) { + usersByName.remove(user.name) + usersBySessionId.remove(session.id) + } + return user + } +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/UserSession.kt b/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/UserSession.kt new file mode 100644 index 0000000..eadae93 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/kurento/recording/UserSession.kt @@ -0,0 +1,63 @@ +package io.openfuture.openmessenger.kurento.recording + +import com.google.gson.JsonObject +import org.kurento.client.IceCandidate +import org.kurento.client.WebRtcEndpoint +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.web.socket.TextMessage +import org.springframework.web.socket.WebSocketSession +import java.io.IOException + +class UserSession(val session: WebSocketSession, val name: String) { + var sdpOffer: String? = null + var callingTo: String? = null + var callingFrom: String? = null + private var webRtcEndpoint: WebRtcEndpoint? = null + var playingWebRtcEndpoint: WebRtcEndpoint? = null + private val candidateList: MutableList = ArrayList() + + @Throws(IOException::class) + fun sendMessage(message: JsonObject) { + log.debug( + "Sending message from user '{}': {}", + name, message + ) + session.sendMessage(TextMessage(message.toString())) + } + + val sessionId: String + get() = session.id + + fun setWebRtcEndpoint(webRtcEndpoint: WebRtcEndpoint?) { + this.webRtcEndpoint = webRtcEndpoint + + if (this.webRtcEndpoint != null) { + for (e in candidateList) { + this.webRtcEndpoint!!.addIceCandidate(e) + } + candidateList.clear() + } + } + + fun addCandidate(candidate: IceCandidate) { + if (this.webRtcEndpoint != null) { + webRtcEndpoint!!.addIceCandidate(candidate) + } else { + candidateList.add(candidate) + } + + if (this.playingWebRtcEndpoint != null) { + playingWebRtcEndpoint!!.addIceCandidate(candidate) + } + } + + fun clear() { + this.webRtcEndpoint = null + candidateList.clear() + } + + companion object { + private val log: Logger = LoggerFactory.getLogger(UserSession::class.java) + } +} diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/AttachmentRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/AttachmentRepository.kt similarity index 58% rename from src/main/kotlin/io/openfuture/openmessanger/repository/AttachmentRepository.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/AttachmentRepository.kt index cc0bbd7..5b1e71c 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/AttachmentRepository.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/AttachmentRepository.kt @@ -1,5 +1,6 @@ -package io.openfuture.openmessanger.repository +package io.openfuture.openmessenger.repository +import io.openfuture.openmessenger.repository.entity.dto.AttachmentResponse import lombok.RequiredArgsConstructor import org.springframework.jdbc.core.namedparam.MapSqlParameterSource import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations @@ -27,4 +28,21 @@ class AttachmentRepository( return keyHolder.keys?.get("id") as Int? } + fun get(id: List): List? { + val sql = """ + select id, name, url from attachment where id=:id; + """.trimIndent() + val parameterSource = MapSqlParameterSource() + .addValue("id", id) + + return jdbcOperations.query(sql, parameterSource) { rs, rowNum -> + AttachmentResponse( + rs.getInt("id"), + rs.getString("name"), + "" + ) + } + + } + } \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/ChatParticipantRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/ChatParticipantRepository.kt new file mode 100644 index 0000000..962b6f8 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/ChatParticipantRepository.kt @@ -0,0 +1,8 @@ +package io.openfuture.openmessenger.repository + +import io.openfuture.openmessenger.repository.entity.ChatParticipant +import org.springframework.data.jpa.repository.JpaRepository + +interface ChatParticipantRepository : JpaRepository { + fun findAllByChatId(chat: Int?): List +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/GroupChatRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/GroupChatRepository.kt similarity index 85% rename from src/main/kotlin/io/openfuture/openmessanger/repository/GroupChatRepository.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/GroupChatRepository.kt index 8a399c7..8e9c985 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/GroupChatRepository.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/GroupChatRepository.kt @@ -1,6 +1,6 @@ -package io.openfuture.openmessanger.repository +package io.openfuture.openmessenger.repository -import io.openfuture.openmessanger.repository.entity.GroupChat +import io.openfuture.openmessenger.repository.entity.GroupChat import org.springframework.data.jpa.repository.JpaRepository import org.springframework.data.jpa.repository.Query import java.util.* diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/GroupParticipantRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/GroupParticipantRepository.kt similarity index 72% rename from src/main/kotlin/io/openfuture/openmessanger/repository/GroupParticipantRepository.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/GroupParticipantRepository.kt index deef41d..899d1d0 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/GroupParticipantRepository.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/GroupParticipantRepository.kt @@ -1,7 +1,7 @@ -package io.openfuture.openmessanger.repository +package io.openfuture.openmessenger.repository -import io.openfuture.openmessanger.repository.entity.GroupChat -import io.openfuture.openmessanger.repository.entity.GroupParticipant +import io.openfuture.openmessenger.repository.entity.GroupChat +import io.openfuture.openmessenger.repository.entity.GroupParticipant import org.springframework.data.jpa.repository.JpaRepository import java.util.* diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/MeetingNoteRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/MeetingNoteRepository.kt new file mode 100644 index 0000000..3b64856 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/MeetingNoteRepository.kt @@ -0,0 +1,7 @@ +package io.openfuture.openmessenger.repository + +import io.openfuture.openmessenger.repository.entity.MeetingNoteEntity +import org.springframework.data.jpa.repository.JpaRepository + +interface MeetingNoteRepository : JpaRepository { +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/MessageAttachmentRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/MessageAttachmentRepository.kt similarity index 66% rename from src/main/kotlin/io/openfuture/openmessanger/repository/MessageAttachmentRepository.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/MessageAttachmentRepository.kt index fc02a23..5bc3bd5 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/MessageAttachmentRepository.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/MessageAttachmentRepository.kt @@ -1,6 +1,6 @@ -package io.openfuture.openmessanger.repository +package io.openfuture.openmessenger.repository -import io.openfuture.openmessanger.repository.entity.MessageAttachment +import io.openfuture.openmessenger.repository.entity.MessageAttachment import org.springframework.data.jpa.repository.JpaRepository interface MessageAttachmentRepository : JpaRepository { diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/MessageJdbcRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/MessageJdbcRepository.kt new file mode 100644 index 0000000..358cdaf --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/MessageJdbcRepository.kt @@ -0,0 +1,32 @@ +package io.openfuture.openmessenger.repository + +import io.openfuture.openmessenger.repository.entity.Message +import org.springframework.data.domain.Sort +import org.springframework.data.jpa.repository.Query +import org.springframework.data.repository.CrudRepository +import org.springframework.data.repository.query.Param +import org.springframework.stereotype.Repository +import java.time.LocalDateTime + +@Repository +interface MessageJdbcRepository : CrudRepository { + + @Query("SELECT m FROM Message m WHERE m.privateChatId = :privateChatId AND m.sentAt BETWEEN :start AND :end") + fun findByPrivateChatIdAndSentAtBetween( + @Param("privateChatId") privateChatId: Int, + @Param("start") start: LocalDateTime, + @Param("end") end: LocalDateTime, + sort: Sort + ): List + + @Query(""" + SELECT m FROM Message m WHERE m.groupChatId = :groupChatId AND m.sentAt BETWEEN :start AND :end + """) + fun findByGroupChatIdAndSentAtBetween( + @Param("groupChatId") groupChatId: Int, + @Param("start") start: LocalDateTime, + @Param("end") end: LocalDateTime, + sort: Sort + ): List + +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/MessageRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/MessageRepository.kt similarity index 95% rename from src/main/kotlin/io/openfuture/openmessanger/repository/MessageRepository.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/MessageRepository.kt index 652b81f..c2938a7 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/MessageRepository.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/MessageRepository.kt @@ -1,11 +1,12 @@ -package io.openfuture.openmessanger.repository +package io.openfuture.openmessenger.repository -import io.openfuture.openmessanger.repository.entity.MessageContentType -import io.openfuture.openmessanger.repository.entity.MessageEntity +import io.openfuture.openmessenger.repository.entity.MessageContentType +import io.openfuture.openmessenger.repository.entity.MessageEntity import lombok.RequiredArgsConstructor import org.springframework.jdbc.core.RowMapper import org.springframework.jdbc.core.namedparam.MapSqlParameterSource import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations +import org.springframework.jdbc.support.GeneratedKeyHolder import org.springframework.stereotype.Repository import java.sql.ResultSet @@ -159,7 +160,7 @@ class MessageRepository( ) } - fun save(message: MessageEntity) { + fun save(message: MessageEntity): Int { val sql = """ INSERT INTO message(private_chat_id, group_chat_id, body, content_type, sender, recipient, received_at, sent_at) VALUES (:privateChatId, :groupChatId, :body, :content_type, :sender, :recipient, :receivedAt, :sentAt) @@ -174,7 +175,11 @@ class MessageRepository( .addValue("content_type", message.contentType.name) .addValue("receivedAt", message.receivedAt) .addValue("sentAt", message.sentAt) - jdbcOperations!!.update(sql, parameterSource) + + val keyHolder = GeneratedKeyHolder() + jdbcOperations.update(sql, parameterSource, keyHolder) + + return keyHolder.keys?.get("id") as Int } fun findGroupMessages(username: String?): List { @@ -200,7 +205,7 @@ class MessageRepository( """.trimIndent() val parameterSource = MapSqlParameterSource() .addValue("user", username) - return jdbcOperations!!.query( + return jdbcOperations.query( sql, parameterSource, groupMessageSelectMapper() diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/NoteRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/NoteRepository.kt new file mode 100644 index 0000000..72f5129 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/NoteRepository.kt @@ -0,0 +1,9 @@ +package io.openfuture.openmessenger.repository + +import io.openfuture.openmessenger.repository.entity.AssistantNoteEntity +import org.springframework.data.jpa.repository.JpaRepository + +interface NoteRepository : JpaRepository { + fun findAllByAuthorAndChatId(author: String, chatId: Int): List + fun findAllByAuthorAndGroupChatId(author: String, chatId: Int): List +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/PrivateChatRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/PrivateChatRepository.kt similarity index 80% rename from src/main/kotlin/io/openfuture/openmessanger/repository/PrivateChatRepository.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/PrivateChatRepository.kt index 8621e27..f05fed1 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/PrivateChatRepository.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/PrivateChatRepository.kt @@ -1,6 +1,6 @@ -package io.openfuture.openmessanger.repository +package io.openfuture.openmessenger.repository -import io.openfuture.openmessanger.repository.entity.PrivateChat +import io.openfuture.openmessenger.repository.entity.PrivateChat import org.springframework.data.jpa.repository.JpaRepository import org.springframework.data.jpa.repository.Query import org.springframework.data.repository.query.Param @@ -17,12 +17,12 @@ interface PrivateChatRepository : JpaRepository { fun findPrivateChatByParticipants( @Param("sender") sender: String?, @Param("recipient") recipient: String? - ): Optional? + ): PrivateChat? @Query( "SELECT pc FROM PrivateChat pc " + "JOIN pc.chatParticipants cp " + "WHERE cp.username = :username AND pc.type = 'SELF' " ) - fun findSelfChat(username: String?): Optional? + fun findSelfChat(username: String?): PrivateChat? } \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/ReminderRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/ReminderRepository.kt new file mode 100644 index 0000000..0e98a11 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/ReminderRepository.kt @@ -0,0 +1,9 @@ +package io.openfuture.openmessenger.repository + +import io.openfuture.openmessenger.repository.entity.AssistantReminderEntity +import org.springframework.data.jpa.repository.JpaRepository + +interface ReminderRepository : JpaRepository { + fun findAllByAuthorAndChatId(author: String, chatId: Int): List + fun findAllByAuthorAndGroupChatId(author: String, chatId: Int): List +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/TodoRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/TodoRepository.kt new file mode 100644 index 0000000..1bb983b --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/TodoRepository.kt @@ -0,0 +1,9 @@ +package io.openfuture.openmessenger.repository + +import io.openfuture.openmessenger.repository.entity.AssistantTodoEntity +import org.springframework.data.jpa.repository.JpaRepository + +interface TodoRepository : JpaRepository { + fun findAllByAuthorAndChatId(author: String, chatId: Int): List + fun findAllByAuthorAndGroupChatId(author: String, chatId: Int): List +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/UserJpaRepository.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/UserJpaRepository.kt similarity index 61% rename from src/main/kotlin/io/openfuture/openmessanger/repository/UserJpaRepository.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/UserJpaRepository.kt index 0098148..03d16b2 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/UserJpaRepository.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/UserJpaRepository.kt @@ -1,6 +1,6 @@ -package io.openfuture.openmessanger.repository +package io.openfuture.openmessenger.repository -import io.openfuture.openmessanger.repository.entity.User +import io.openfuture.openmessenger.repository.entity.User import org.springframework.data.jpa.repository.JpaRepository interface UserJpaRepository : JpaRepository { diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/entity/AssistantNoteEntity.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/AssistantNoteEntity.kt new file mode 100644 index 0000000..0024c6e --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/AssistantNoteEntity.kt @@ -0,0 +1,48 @@ +package io.openfuture.openmessenger.repository.entity + +import jakarta.persistence.* +import java.time.LocalDateTime +import java.time.LocalDateTime.now + + +@Entity +@Table(name = "assistant_notes") +class AssistantNoteEntity() { + constructor( + author: String?, + chatId: Int?, + groupChatId: Int?, + members: String?, + recipient: String?, + generatedAt: LocalDateTime = now(), + version: Int = 1, + startTime: LocalDateTime?, + endTime: LocalDateTime?, + notes: String? + ): this() { + this.author = author + this.chatId = chatId + this.groupChatId = groupChatId + this.members = members + this.recipient = recipient + this.generatedAt = generatedAt + this.version = version + this.startTime = startTime + this.endTime = endTime + this.notes = notes + } + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + var id: Long? = null + var author: String? = null + var chatId: Int? = null + var groupChatId: Int? = null + var members: String? = null + var recipient: String? = null + var generatedAt: LocalDateTime = now() + var version: Int = 1 + var startTime: LocalDateTime? = null + var endTime: LocalDateTime? = null + var notes: String? = null +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/entity/AssistantReminderEntity.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/AssistantReminderEntity.kt new file mode 100644 index 0000000..d5020b0 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/AssistantReminderEntity.kt @@ -0,0 +1,47 @@ +package io.openfuture.openmessenger.repository.entity + +import jakarta.persistence.* +import java.time.LocalDateTime +import java.time.LocalDateTime.now + +@Entity +@Table(name = "assistant_reminders") +class AssistantReminderEntity() { + constructor( + author: String?, + chatId: Int?, + groupChatId: Int?, + members: String?, + recipient: String?, + generatedAt: LocalDateTime = now(), + version: Int = 1, + startTime: LocalDateTime?, + endTime: LocalDateTime?, + reminders: String? + ): this() { + this.author = author + this.chatId = chatId + this.groupChatId = groupChatId + this.members = members + this.recipient = recipient + this.generatedAt = generatedAt + this.version = version + this.startTime = startTime + this.endTime = endTime + this.reminders = reminders + } + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + var id: Long? = null + var author: String? = null + var chatId: Int? = null + var groupChatId: Int? = null + var members: String? = null + var recipient: String? = null + var generatedAt: LocalDateTime = now() + var version: Int = 1 + var startTime: LocalDateTime? = null + var endTime: LocalDateTime? = null + var reminders: String? = null +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/entity/AssistantTodoEntity.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/AssistantTodoEntity.kt new file mode 100644 index 0000000..dc24279 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/AssistantTodoEntity.kt @@ -0,0 +1,47 @@ +package io.openfuture.openmessenger.repository.entity + +import jakarta.persistence.* +import java.time.LocalDateTime +import java.time.LocalDateTime.now + +@Entity +@Table(name = "assistant_todos") +class AssistantTodoEntity() { + constructor( + author: String?, + chatId: Int?, + groupChatId: Int?, + members: String?, + recipient: String?, + generatedAt: LocalDateTime = now(), + version: Int = 1, + startTime: LocalDateTime?, + endTime: LocalDateTime?, + todos: String? + ): this() { + this.author = author + this.chatId = chatId + this.groupChatId = groupChatId + this.members = members + this.recipient = recipient + this.generatedAt = generatedAt + this.version = version + this.startTime = startTime + this.endTime = endTime + this.todos = todos + } + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + var id: Long? = null + var author: String? = null + var chatId: Int? = null + var groupChatId: Int? = null + var members: String? = null + var recipient: String? = null + var generatedAt: LocalDateTime = now() + var version: Int = 1 + var startTime: LocalDateTime? = null + var endTime: LocalDateTime? = null + var todos: String? = null +} diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/Attachment.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/Attachment.kt similarity index 86% rename from src/main/kotlin/io/openfuture/openmessanger/repository/entity/Attachment.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/entity/Attachment.kt index 792ddb2..546c134 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/Attachment.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/Attachment.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.repository.entity +package io.openfuture.openmessenger.repository.entity import jakarta.persistence.* import java.time.LocalDateTime diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/ChatParticipant.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/ChatParticipant.kt similarity index 55% rename from src/main/kotlin/io/openfuture/openmessanger/repository/entity/ChatParticipant.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/entity/ChatParticipant.kt index 3b2827c..f0e816f 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/ChatParticipant.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/ChatParticipant.kt @@ -1,10 +1,10 @@ -package io.openfuture.openmessanger.repository.entity +package io.openfuture.openmessenger.repository.entity import jakarta.persistence.* @Entity @Table(name = "chat_participant") -class ChatParticipant(var chatId: Int?, var username: String?) { +class ChatParticipant(val chatId: Int? = 0, val username: String? = null) { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/GroupChat.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/GroupChat.kt similarity index 94% rename from src/main/kotlin/io/openfuture/openmessanger/repository/entity/GroupChat.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/entity/GroupChat.kt index f55ea66..5f30a43 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/GroupChat.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/GroupChat.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.repository.entity +package io.openfuture.openmessenger.repository.entity import jakarta.persistence.* import java.time.LocalDateTime diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/GroupParticipant.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/GroupParticipant.kt similarity index 94% rename from src/main/kotlin/io/openfuture/openmessanger/repository/entity/GroupParticipant.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/entity/GroupParticipant.kt index c800e7b..e8f1d00 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/GroupParticipant.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/GroupParticipant.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.repository.entity +package io.openfuture.openmessenger.repository.entity import jakarta.persistence.* import java.time.LocalDateTime diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/entity/MeetingNoteEntity.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/MeetingNoteEntity.kt new file mode 100644 index 0000000..faad2d0 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/MeetingNoteEntity.kt @@ -0,0 +1,48 @@ +package io.openfuture.openmessenger.repository.entity + +import jakarta.persistence.* +import java.time.LocalDateTime +import java.time.LocalDateTime.now + + +@Entity +@Table(name = "meeting_notes") +class MeetingNoteEntity() { + constructor( + author: String?, + chatId: Int?, + groupChatId: Int?, + members: String?, + recipient: String?, + generatedAt: LocalDateTime = now(), + version: Int = 1, + startTime: LocalDateTime?, + endTime: LocalDateTime?, + notes: String? + ): this() { + this.author = author + this.chatId = chatId + this.groupChatId = groupChatId + this.members = members + this.recipient = recipient + this.generatedAt = generatedAt + this.version = version + this.startTime = startTime + this.endTime = endTime + this.notes = notes + } + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + var id: Long? = null + var author: String? = null + var chatId: Int? = null + var groupChatId: Int? = null + var members: String? = null + var recipient: String? = null + var generatedAt: LocalDateTime = now() + var version: Int = 1 + var startTime: LocalDateTime? = null + var endTime: LocalDateTime? = null + var notes: String? = null +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/entity/Message.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/Message.kt new file mode 100644 index 0000000..945de6e --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/Message.kt @@ -0,0 +1,21 @@ +package io.openfuture.openmessenger.repository.entity + +import jakarta.persistence.Entity +import jakarta.persistence.Id +import jakarta.persistence.Table +import java.time.LocalDateTime + +@Table(name = "message") +@Entity +class Message { + @Id + var id: Int = 0 + var body: String? = null + var sender: String? = null + var recipient: String? = null + var contentType: String? = null + var receivedAt: LocalDateTime? = null + var sentAt: LocalDateTime? = null + var privateChatId: Int? = null + var groupChatId: Int? = null +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/entity/MessageAttachment.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/MessageAttachment.kt new file mode 100644 index 0000000..bb96482 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/MessageAttachment.kt @@ -0,0 +1,26 @@ +package io.openfuture.openmessenger.repository.entity + +import jakarta.persistence.* + +@Entity +@Table(name = "message_attachment") +class MessageAttachment() { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + val id: Int? = null + + @Column(name = "attachment_id") + var attachmentId: Int? = null + + @Column(name = "message_id") + var messageId: Int? = null + + constructor( + attachmentIds : Int?, + messageIds : Int? + ) : this() { + this.attachmentId = attachmentIds + this.messageId = messageIds + } +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/MessageContentType.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/MessageContentType.kt similarity index 50% rename from src/main/kotlin/io/openfuture/openmessanger/repository/entity/MessageContentType.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/entity/MessageContentType.kt index eca5139..2b1c48f 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/MessageContentType.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/MessageContentType.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.repository.entity +package io.openfuture.openmessenger.repository.entity enum class MessageContentType { TEXT, ATTACHMENT diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/MessageEntity.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/MessageEntity.kt similarity index 96% rename from src/main/kotlin/io/openfuture/openmessanger/repository/entity/MessageEntity.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/entity/MessageEntity.kt index d3bf466..6324eb3 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/MessageEntity.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/MessageEntity.kt @@ -1,9 +1,9 @@ -package io.openfuture.openmessanger.repository.entity +package io.openfuture.openmessenger.repository.entity import java.time.LocalDateTime class MessageEntity { - var id = 0 + var id: Int = 0 var body: String? var sender: String var recipient: String? = null diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/PrivateChat.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/PrivateChat.kt similarity index 66% rename from src/main/kotlin/io/openfuture/openmessanger/repository/entity/PrivateChat.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/entity/PrivateChat.kt index 227cdb9..20de593 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/PrivateChat.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/PrivateChat.kt @@ -1,13 +1,10 @@ -package io.openfuture.openmessanger.repository.entity +package io.openfuture.openmessenger.repository.entity import jakarta.persistence.* @Entity @Table(name = "private_chat") -class PrivateChat( - @Column(name = "type") - val type: String -) { +class PrivateChat(val type: String? = null) { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) val id: Int? = null diff --git a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/User.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/User.kt similarity index 67% rename from src/main/kotlin/io/openfuture/openmessanger/repository/entity/User.kt rename to src/main/kotlin/io/openfuture/openmessenger/repository/entity/User.kt index 11bf9f2..7ac124b 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/repository/entity/User.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/User.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.repository.entity +package io.openfuture.openmessenger.repository.entity import jakarta.persistence.* import java.time.ZonedDateTime @@ -9,7 +9,7 @@ class User(email: String? = null, firstName: String? = null, lastName: String? = @Id @GeneratedValue(strategy = GenerationType.IDENTITY) - private val id = 0 + val id = 0 @Column(name = "email", unique = false) var email: String? = null @@ -21,17 +21,17 @@ class User(email: String? = null, firstName: String? = null, lastName: String? = var lastName: String? = null @Column(name = "phone_number") - private val phoneNumber: String? = null + val phoneNumber: String? = null @Column(name = "registered_at") - private val registeredAt: ZonedDateTime = ZonedDateTime.now() + val registeredAt: ZonedDateTime = ZonedDateTime.now() @Column(name = "last_login") - private val lastLogin: ZonedDateTime? = null + val lastLogin: ZonedDateTime? = null @Column(name = "avatar") - private val avatar: String? = null + val avatar: String? = null @Column(name = "active") - private val active = true + val active = true } \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/repository/entity/dto/AttachmentResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/dto/AttachmentResponse.kt new file mode 100644 index 0000000..8461bc9 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/repository/entity/dto/AttachmentResponse.kt @@ -0,0 +1,7 @@ +package io.openfuture.openmessenger.repository.entity.dto + +data class AttachmentResponse( + val id: Int, + val name: String, + val url: String +) \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/security/AwsCognitoTokenFilter.kt b/src/main/kotlin/io/openfuture/openmessenger/security/AwsCognitoTokenFilter.kt similarity index 78% rename from src/main/kotlin/io/openfuture/openmessanger/security/AwsCognitoTokenFilter.kt rename to src/main/kotlin/io/openfuture/openmessenger/security/AwsCognitoTokenFilter.kt index 8501586..54b8bf8 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/security/AwsCognitoTokenFilter.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/security/AwsCognitoTokenFilter.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.security +package io.openfuture.openmessenger.security import jakarta.servlet.FilterChain import jakarta.servlet.ServletException @@ -20,7 +20,9 @@ class AwsCognitoTokenFilter( defaultFilterProcessesUrl: String?, authenticationManager: AuthenticationManager?, loginUrl: String?, - signupUrl: String? + signupUrl: String?, + attachmentDownloadUrl: String?, + allowedPages: List, ) : AbstractAuthenticationProcessingFilter(defaultFilterProcessesUrl) { companion object{ private val log = LoggerFactory.getLogger(AwsCognitoTokenFilter::class.java) @@ -28,21 +30,26 @@ class AwsCognitoTokenFilter( private val loginRequestMatcher: RequestMatcher = AntPathRequestMatcher(loginUrl) private val signupRequestMatcher: RequestMatcher = AntPathRequestMatcher(signupUrl) + private val attachmentDownloadRequestMatcher: RequestMatcher = AntPathRequestMatcher(attachmentDownloadUrl) + private val allowedPagesRequestMatchers: List = + allowedPages.map { AntPathRequestMatcher(it) } init { setAuthenticationManager(authenticationManager) } override fun requiresAuthentication(request: HttpServletRequest, response: HttpServletResponse): Boolean { - return !loginRequestMatcher.matches(request) && !signupRequestMatcher.matches(request) + return !loginRequestMatcher.matches(request) && + !signupRequestMatcher.matches(request) && + !attachmentDownloadRequestMatcher.matches(request) && + allowedPagesRequestMatchers.all { !it.matches(request) } } @Throws(AuthenticationException::class, IOException::class, ServletException::class) override fun attemptAuthentication(request: HttpServletRequest, response: HttpServletResponse): Authentication { val header = request.getHeader(HttpHeaders.AUTHORIZATION) - log.info("Header : $header") if (header == null || header.isEmpty() || header.length == 6) { - log.info("Token is not provided") + log.info("Url=${request.requestURL} Token is not provided") throw AuthenticationServiceException("Token is missing") } val token = header.substring("Bearer ".length) diff --git a/src/main/kotlin/io/openfuture/openmessanger/security/CognitoAuthenticationProvider.kt b/src/main/kotlin/io/openfuture/openmessenger/security/CognitoAuthenticationProvider.kt similarity index 96% rename from src/main/kotlin/io/openfuture/openmessanger/security/CognitoAuthenticationProvider.kt rename to src/main/kotlin/io/openfuture/openmessenger/security/CognitoAuthenticationProvider.kt index 8389a5a..a4d6fa9 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/security/CognitoAuthenticationProvider.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/security/CognitoAuthenticationProvider.kt @@ -1,9 +1,8 @@ -package io.openfuture.openmessanger.security +package io.openfuture.openmessenger.security import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import io.jsonwebtoken.* -import io.openfuture.openmessanger.security.CognitoAuthenticationToken import lombok.SneakyThrows import lombok.extern.slf4j.Slf4j import org.springframework.beans.factory.annotation.Value diff --git a/src/main/kotlin/io/openfuture/openmessanger/security/CognitoAuthenticationToken.kt b/src/main/kotlin/io/openfuture/openmessenger/security/CognitoAuthenticationToken.kt similarity index 94% rename from src/main/kotlin/io/openfuture/openmessanger/security/CognitoAuthenticationToken.kt rename to src/main/kotlin/io/openfuture/openmessenger/security/CognitoAuthenticationToken.kt index fff502b..7b21b4a 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/security/CognitoAuthenticationToken.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/security/CognitoAuthenticationToken.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.security +package io.openfuture.openmessenger.security import org.springframework.security.authentication.AbstractAuthenticationToken import org.springframework.security.core.GrantedAuthority diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/AssistantService.kt b/src/main/kotlin/io/openfuture/openmessenger/service/AssistantService.kt new file mode 100644 index 0000000..cd33e78 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/AssistantService.kt @@ -0,0 +1,19 @@ +package io.openfuture.openmessenger.service + +import io.openfuture.openmessenger.assistant.model.ConversationNotes +import io.openfuture.openmessenger.assistant.model.Reminder +import io.openfuture.openmessenger.assistant.model.Todos +import io.openfuture.openmessenger.service.dto.AssistantRequest +import io.openfuture.openmessenger.service.dto.GetAllNotesRequest +import io.openfuture.openmessenger.service.dto.GetAllRemindersRequest +import io.openfuture.openmessenger.service.dto.GetAllTodosRequest + +interface AssistantService { + fun generateNotes(assistantRequest: AssistantRequest): ConversationNotes? + fun generateReminder(assistantRequest: AssistantRequest): Reminder + fun generateTodos(assistantRequest: AssistantRequest): Todos + + fun getAllNotes(getAllNotesRequest: GetAllNotesRequest): List + fun getAllTodos(getAllTodosRequest: GetAllTodosRequest): List + fun getAllReminders(getAllRemindersRequest: GetAllRemindersRequest): List +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/AttachmentService.kt b/src/main/kotlin/io/openfuture/openmessenger/service/AttachmentService.kt similarity index 57% rename from src/main/kotlin/io/openfuture/openmessanger/service/AttachmentService.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/AttachmentService.kt index cb9a3b6..bbaa421 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/AttachmentService.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/AttachmentService.kt @@ -1,16 +1,21 @@ -package io.openfuture.openmessanger.service +package io.openfuture.openmessenger.service import org.springframework.web.multipart.MultipartFile import java.io.IOException +import java.io.InputStream import kotlin.Throws interface AttachmentService { @Throws(IOException::class) fun upload(file: MultipartFile) + fun upload(name: String, fileInputStream: InputStream) + @Throws(IOException::class) fun uploadAndReturnId(file: MultipartFile): Int @Throws(IOException::class) - fun download(fileName: String?): ByteArray? + fun download(fileName: String?, bucket: String): ByteArray? + + fun downloadById(id: Int): ByteArray? } \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/CognitoUserService.kt b/src/main/kotlin/io/openfuture/openmessenger/service/CognitoUserService.kt similarity index 91% rename from src/main/kotlin/io/openfuture/openmessanger/service/CognitoUserService.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/CognitoUserService.kt index 4c4af59..89b2e75 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/CognitoUserService.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/CognitoUserService.kt @@ -1,7 +1,7 @@ -package io.openfuture.openmessanger.service +package io.openfuture.openmessenger.service import com.amazonaws.services.cognitoidp.model.* -import io.openfuture.openmessanger.service.dto.UserSignUpRequest +import io.openfuture.openmessenger.service.dto.UserSignUpRequest import java.util.* interface CognitoUserService { diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/GroupChatService.kt b/src/main/kotlin/io/openfuture/openmessenger/service/GroupChatService.kt similarity index 57% rename from src/main/kotlin/io/openfuture/openmessanger/service/GroupChatService.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/GroupChatService.kt index 7bcfbb1..dad9630 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/GroupChatService.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/GroupChatService.kt @@ -1,9 +1,9 @@ -package io.openfuture.openmessanger.service +package io.openfuture.openmessenger.service -import io.openfuture.openmessanger.repository.entity.GroupChat -import io.openfuture.openmessanger.web.request.group.AddParticipantsRequest -import io.openfuture.openmessanger.web.request.group.CreateGroupRequest -import io.openfuture.openmessanger.web.request.group.RemoveParticipantsRequest +import io.openfuture.openmessenger.repository.entity.GroupChat +import io.openfuture.openmessenger.web.request.group.AddParticipantsRequest +import io.openfuture.openmessenger.web.request.group.CreateGroupRequest +import io.openfuture.openmessenger.web.request.group.RemoveParticipantsRequest interface GroupChatService { fun get(groupId: Int): GroupChat? diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/MessageService.kt b/src/main/kotlin/io/openfuture/openmessenger/service/MessageService.kt similarity index 59% rename from src/main/kotlin/io/openfuture/openmessanger/service/MessageService.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/MessageService.kt index 90cfa24..5c5c216 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/MessageService.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/MessageService.kt @@ -1,11 +1,11 @@ -package io.openfuture.openmessanger.service +package io.openfuture.openmessenger.service -import io.openfuture.openmessanger.web.request.GroupMessageRequest -import io.openfuture.openmessanger.web.request.MessageRequest -import io.openfuture.openmessanger.web.request.MessageToAssistantRequest -import io.openfuture.openmessanger.web.response.GroupMessageResponse -import io.openfuture.openmessanger.web.response.LastMessage -import io.openfuture.openmessanger.web.response.MessageResponse +import io.openfuture.openmessenger.web.request.GroupMessageRequest +import io.openfuture.openmessenger.web.request.MessageRequest +import io.openfuture.openmessenger.web.request.MessageToAssistantRequest +import io.openfuture.openmessenger.web.response.GroupMessageResponse +import io.openfuture.openmessenger.web.response.LastMessage +import io.openfuture.openmessenger.web.response.MessageResponse interface MessageService { fun sendMessage(request: MessageRequest) diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/PrivateChatService.kt b/src/main/kotlin/io/openfuture/openmessenger/service/PrivateChatService.kt new file mode 100644 index 0000000..728c0ee --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/PrivateChatService.kt @@ -0,0 +1,8 @@ +package io.openfuture.openmessenger.service + +import io.openfuture.openmessenger.repository.entity.ChatParticipant + +interface PrivateChatService { + fun getOtherUser(username: String, chatId: Int?): ChatParticipant? + fun getParticipants(chatId: Int?): List? +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/RecordingManagementService.kt b/src/main/kotlin/io/openfuture/openmessenger/service/RecordingManagementService.kt new file mode 100644 index 0000000..5a26379 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/RecordingManagementService.kt @@ -0,0 +1,6 @@ +package io.openfuture.openmessenger.service + +interface RecordingManagementService { + fun list() + fun uploadToS3(fileUri: String): Int +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/SpeechToTextService.kt b/src/main/kotlin/io/openfuture/openmessenger/service/SpeechToTextService.kt new file mode 100644 index 0000000..bb491fb --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/SpeechToTextService.kt @@ -0,0 +1,6 @@ +package io.openfuture.openmessenger.service + +interface SpeechToTextService { + fun extractTranscript(attachmentId: Int): String + fun getTranscript(attachmentId: String): String +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/TranscribeService.kt b/src/main/kotlin/io/openfuture/openmessenger/service/TranscribeService.kt new file mode 100644 index 0000000..0f8dbe8 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/TranscribeService.kt @@ -0,0 +1,5 @@ +package io.openfuture.openmessenger.service + +interface TranscribeService { + fun extractText(filename: String): String? +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/UserAuthService.kt b/src/main/kotlin/io/openfuture/openmessenger/service/UserAuthService.kt similarity index 77% rename from src/main/kotlin/io/openfuture/openmessanger/service/UserAuthService.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/UserAuthService.kt index fb67121..f2f1138 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/UserAuthService.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/UserAuthService.kt @@ -1,10 +1,10 @@ -package io.openfuture.openmessanger.service +package io.openfuture.openmessenger.service import com.amazonaws.services.cognitoidp.model.AdminListUserAuthEventsResult import com.amazonaws.services.cognitoidp.model.ForgotPasswordResult -import io.openfuture.openmessanger.service.dto.* -import io.openfuture.openmessanger.service.response.UserResponse -import io.openfuture.openmessanger.web.response.AuthenticatedResponse +import io.openfuture.openmessenger.service.dto.* +import io.openfuture.openmessenger.service.response.UserResponse +import io.openfuture.openmessenger.web.response.AuthenticatedResponse import javax.validation.constraints.NotNull interface UserAuthService { diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/UserService.kt b/src/main/kotlin/io/openfuture/openmessenger/service/UserService.kt similarity index 53% rename from src/main/kotlin/io/openfuture/openmessanger/service/UserService.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/UserService.kt index 342da3f..41c1e7b 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/UserService.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/UserService.kt @@ -1,8 +1,8 @@ -package io.openfuture.openmessanger.service +package io.openfuture.openmessenger.service -import io.openfuture.openmessanger.repository.entity.User -import io.openfuture.openmessanger.web.request.user.UserDetailsRequest -import io.openfuture.openmessanger.web.response.UserDetailsResponse +import io.openfuture.openmessenger.repository.entity.User +import io.openfuture.openmessenger.web.request.user.UserDetailsRequest +import io.openfuture.openmessenger.web.response.UserDetailsResponse interface UserService { fun getAllRecipientsBySender(username: String?): Collection? diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/VideoService.kt b/src/main/kotlin/io/openfuture/openmessenger/service/VideoService.kt new file mode 100644 index 0000000..99c0e2f --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/VideoService.kt @@ -0,0 +1,5 @@ +package io.openfuture.openmessenger.service + +interface VideoService { + fun convertToAudio() +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/dto/AssistantRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/service/dto/AssistantRequest.kt new file mode 100644 index 0000000..99cee80 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/dto/AssistantRequest.kt @@ -0,0 +1,10 @@ +package io.openfuture.openmessenger.service.dto + +import java.time.LocalDateTime + +data class AssistantRequest( + val chatId: Int, + val isGroup: Boolean, + val startTime: LocalDateTime, + val endTime: LocalDateTime +) diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/dto/AuthenticatedChallengeRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/service/dto/AuthenticatedChallengeRequest.kt similarity index 85% rename from src/main/kotlin/io/openfuture/openmessanger/service/dto/AuthenticatedChallengeRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/dto/AuthenticatedChallengeRequest.kt index 8f4234e..1fb797c 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/dto/AuthenticatedChallengeRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/dto/AuthenticatedChallengeRequest.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.service.dto +package io.openfuture.openmessenger.service.dto import javax.validation.constraints.NotBlank diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/dto/GetAllNotesRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/service/dto/GetAllNotesRequest.kt new file mode 100644 index 0000000..60dea24 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/dto/GetAllNotesRequest.kt @@ -0,0 +1,6 @@ +package io.openfuture.openmessenger.service.dto + +data class GetAllNotesRequest( + val chatId: Int, + val isGroup: Boolean, +) \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/dto/GetAllRemindersRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/service/dto/GetAllRemindersRequest.kt new file mode 100644 index 0000000..4d3f759 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/dto/GetAllRemindersRequest.kt @@ -0,0 +1,6 @@ +package io.openfuture.openmessenger.service.dto + +data class GetAllRemindersRequest( + val chatId: Int, + val isGroup: Boolean, +) \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/dto/GetAllTodosRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/service/dto/GetAllTodosRequest.kt new file mode 100644 index 0000000..49e8c4d --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/dto/GetAllTodosRequest.kt @@ -0,0 +1,6 @@ +package io.openfuture.openmessenger.service.dto + +data class GetAllTodosRequest( + val chatId: Int, + val isGroup: Boolean, +) \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/dto/LoginRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/service/dto/LoginRequest.kt similarity index 76% rename from src/main/kotlin/io/openfuture/openmessanger/service/dto/LoginRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/dto/LoginRequest.kt index 2677a57..43c6c55 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/dto/LoginRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/dto/LoginRequest.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.service.dto +package io.openfuture.openmessenger.service.dto import javax.validation.constraints.NotBlank diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/dto/LoginSmsVerifyRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/service/dto/LoginSmsVerifyRequest.kt similarity index 81% rename from src/main/kotlin/io/openfuture/openmessanger/service/dto/LoginSmsVerifyRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/dto/LoginSmsVerifyRequest.kt index 1cd083a..6044155 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/dto/LoginSmsVerifyRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/dto/LoginSmsVerifyRequest.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.service.dto +package io.openfuture.openmessenger.service.dto import javax.validation.constraints.NotBlank diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/dto/RefreshTokenRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/service/dto/RefreshTokenRequest.kt similarity index 54% rename from src/main/kotlin/io/openfuture/openmessanger/service/dto/RefreshTokenRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/dto/RefreshTokenRequest.kt index fbd648c..c99620c 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/dto/RefreshTokenRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/dto/RefreshTokenRequest.kt @@ -1,3 +1,3 @@ -package io.openfuture.openmessanger.service.dto +package io.openfuture.openmessenger.service.dto data class RefreshTokenRequest(var refreshToken: String) \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/dto/UserSignUpRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/service/dto/UserSignUpRequest.kt similarity index 91% rename from src/main/kotlin/io/openfuture/openmessanger/service/dto/UserSignUpRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/dto/UserSignUpRequest.kt index bf27ffd..72f381a 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/dto/UserSignUpRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/dto/UserSignUpRequest.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.service.dto +package io.openfuture.openmessenger.service.dto import javax.validation.constraints.Email import javax.validation.constraints.NotBlank diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/dto/transcript/TranscriptionResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/service/dto/transcript/TranscriptionResponse.kt new file mode 100644 index 0000000..be31d47 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/dto/transcript/TranscriptionResponse.kt @@ -0,0 +1,49 @@ +package io.openfuture.openmessenger.service.dto.transcript + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonProperty + +data class TranscriptionResponse( + val jobName: String, + val accountId: String, + val status: String, + val results: Results +) + +data class Results( + val transcripts: List = emptyList(), + @JsonIgnore + val items: List = emptyList(), + @JsonIgnore + @JsonProperty("audio_segments") + val audioSegments: List = emptyList() +) + +data class Transcript( + val transcript: String +) + +data class Item( + val id: String, + val type: String, + val alternatives: List, + @JsonProperty("start_time") + val startTime: String, + @JsonProperty("end_time") + val endTime: String +) + +data class Alternative( + val confidence: String, + val content: String +) + +data class AudioSegment( + val id: String, + val transcript: String, + @JsonProperty("start_time") + val startTime: String, + @JsonProperty("end_time") + val endTime: String, + val items: List +) \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/impl/AmazonTranscribeServiceImpl.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/AmazonTranscribeServiceImpl.kt new file mode 100644 index 0000000..ac32583 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/AmazonTranscribeServiceImpl.kt @@ -0,0 +1,60 @@ +package io.openfuture.openmessenger.service.impl + +import com.amazonaws.services.transcribe.AmazonTranscribe +import com.amazonaws.services.transcribe.model.* +import io.openfuture.openmessenger.configuration.AwsConfig +import io.openfuture.openmessenger.service.TranscribeService +import org.springframework.stereotype.Service +import java.util.* + + +@Service +class AmazonTranscribeServiceImpl( + private val amazonTranscribe: AmazonTranscribe, + private val awsConfig: AwsConfig +): TranscribeService { + + override fun extractText(filename: String): String? { + + val transcriptionJobName = "TranscriptionJob1-$filename-${UUID.randomUUID()}" + val outputS3BucketName = awsConfig.transcriptsBucket + val inputS3BucketName = awsConfig.recordingsBucket + + val s3Filename = "s3://$inputS3BucketName/$filename" + val myMedia: Media = Media().withMediaFileUri(s3Filename) + + val request: StartTranscriptionJobRequest = StartTranscriptionJobRequest() + .withTranscriptionJobName(transcriptionJobName) + .withLanguageCode(LanguageCode.EnUS) + .withMedia(myMedia) + .withOutputBucketName(outputS3BucketName) + + println(request) + val startJobResponse: StartTranscriptionJobResult = amazonTranscribe.startTranscriptionJob(request) + + println("Created the transcription job") + println(startJobResponse.transcriptionJob) + + val getJobRequest: GetTranscriptionJobRequest = GetTranscriptionJobRequest() + .withTranscriptionJobName(transcriptionJobName) + + var jobStatus: String + var getJobResponse: GetTranscriptionJobResult + do { + getJobResponse = amazonTranscribe.getTranscriptionJob(getJobRequest) + jobStatus = getJobResponse.transcriptionJob.transcriptionJobStatus + println("Current status: $jobStatus") + if (jobStatus == TranscriptionJobStatus.COMPLETED.toString()) { + println("Transcription job completed") + println("Transcription: ${getJobResponse.transcriptionJob.transcript.transcriptFileUri}") + } else if (jobStatus == TranscriptionJobStatus.FAILED.toString()) { + println("Transcription job failed: ${getJobResponse.transcriptionJob.failureReason}") + break + } + Thread.sleep(5000) + } while (jobStatus == TranscriptionJobStatus.IN_PROGRESS.toString()) + + return getJobResponse.transcriptionJob.transcript.transcriptFileUri + } + +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/impl/AssistantServiceImpl.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/AssistantServiceImpl.kt new file mode 100644 index 0000000..6fe58f1 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/AssistantServiceImpl.kt @@ -0,0 +1,323 @@ +package io.openfuture.openmessenger.service.impl + +import com.fasterxml.jackson.databind.DeserializationConfig +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import io.openfuture.openmessenger.assistant.gemini.GeminiService +import io.openfuture.openmessenger.assistant.model.* +import io.openfuture.openmessenger.repository.MessageJdbcRepository +import io.openfuture.openmessenger.repository.NoteRepository +import io.openfuture.openmessenger.repository.ReminderRepository +import io.openfuture.openmessenger.repository.TodoRepository +import io.openfuture.openmessenger.repository.entity.AssistantNoteEntity +import io.openfuture.openmessenger.repository.entity.AssistantReminderEntity +import io.openfuture.openmessenger.repository.entity.AssistantTodoEntity +import io.openfuture.openmessenger.repository.entity.Message +import io.openfuture.openmessenger.service.AssistantService +import io.openfuture.openmessenger.service.GroupChatService +import io.openfuture.openmessenger.service.PrivateChatService +import io.openfuture.openmessenger.service.UserAuthService +import io.openfuture.openmessenger.service.dto.AssistantRequest +import io.openfuture.openmessenger.service.dto.GetAllNotesRequest +import io.openfuture.openmessenger.service.dto.GetAllRemindersRequest +import io.openfuture.openmessenger.service.dto.GetAllTodosRequest +import io.openfuture.openmessenger.service.response.UserResponse +import org.springframework.data.domain.Sort +import org.springframework.stereotype.Service +import java.time.LocalDateTime + +@Service +class AssistantServiceImpl( + val geminiService: GeminiService, + val groupChatService: GroupChatService, + val privateChatService: PrivateChatService, + val messageJdbcRepository: MessageJdbcRepository, + val userAuthService: UserAuthService, + val noteRepository: NoteRepository, + val todoRepository: TodoRepository, + val reminderRepository: ReminderRepository +) : AssistantService { + + override fun generateNotes(assistantRequest: AssistantRequest): ConversationNotes? { + val current = userAuthService.current() + + val participants: List? = getParticipants(assistantRequest) + + getRecipient(current, assistantRequest) + + val conversation = getConversation(assistantRequest) + + val result = geminiService.chat("Retrieve a short key notes separated with * from a following conversation related to participant ${current.email}. s$conversation") + + val objectMapper = jacksonObjectMapper() + + val assistantNoteEntity = AssistantNoteEntity( + current.email, + if (assistantRequest.isGroup) null else assistantRequest.chatId, + if (assistantRequest.isGroup) assistantRequest.chatId else null, + objectMapper.writeValueAsString(participants), + getRecipient(current, assistantRequest), + LocalDateTime.now(), + 1, + assistantRequest.startTime, + assistantRequest.endTime, + objectMapper.writeValueAsString(result!!.split("*").filter { s: String -> s.isNotEmpty() }) + ) + noteRepository.save(assistantNoteEntity) + + return ConversationNotes( + if (assistantRequest.isGroup) null else assistantRequest.chatId, + if (assistantRequest.isGroup) assistantRequest.chatId else null, + participants, + getRecipient(current, assistantRequest), + LocalDateTime.now(), + 1, + assistantRequest.startTime, + assistantRequest.endTime, + result.split("*").filter { s: String -> s.isNotEmpty() } + ) + } + + private fun getRecipient(current: UserResponse, assistantRequest: AssistantRequest): String? = if (assistantRequest.isGroup) { + null + } else { + privateChatService.getOtherUser(current.email!!, assistantRequest.chatId)?.username + } + + override fun generateReminder(assistantRequest: AssistantRequest): Reminder { + val current = userAuthService.current() + val participants: List? = getParticipants(assistantRequest) + + val objectMapper = jacksonObjectMapper() + objectMapper.registerModule(JavaTimeModule()) + + val conversation = getConversation(assistantRequest) + + var result = geminiService.chat("${PROMPT_FOR_REMINDER.format(current.email)}. Conversation starts here. $conversation") + + println("Result [$result]") + + result = result + ?.replace("[", "") + ?.replace("]", "") + ?.replace("```json", "") + ?.replace("```", "") + + val reminderItemList = objectMapper.readValue>("[$result]") + val defaultObjectMapper = jacksonObjectMapper() + + val assistantReminderEntity = AssistantReminderEntity( + current.email, + if (assistantRequest.isGroup) null else assistantRequest.chatId, + if (assistantRequest.isGroup) assistantRequest.chatId else null, + objectMapper.writeValueAsString(participants), + getRecipient(current, assistantRequest), + LocalDateTime.now(), + 1, + assistantRequest.startTime, + assistantRequest.endTime, + objectMapper.writeValueAsString(reminderItemList) + ) + reminderRepository.save(assistantReminderEntity) + + return Reminder( + if (assistantRequest.isGroup) null else assistantRequest.chatId, + if (assistantRequest.isGroup) assistantRequest.chatId else null, + participants, + getRecipient(current, assistantRequest), + LocalDateTime.now(), + 1, + assistantRequest.startTime, + assistantRequest.endTime, + reminderItemList.ifEmpty { emptyList() } + ) + } + + override fun generateTodos(assistantRequest: AssistantRequest): Todos { + val current = userAuthService.current() + val participants: List? = getParticipants(assistantRequest) + + val objectMapper = jacksonObjectMapper() + objectMapper.registerModule(JavaTimeModule()) + + val conversation = getConversation(assistantRequest) + + println(conversation) + + var result = geminiService.chat("${PROMPT_TODOS.format(current.email)}. Conversation starts here. $conversation") + + println("Result [$result]") + + result = result + ?.replace("[", "") + ?.replace("]", "") + ?.replace("```json", "") + ?.replace("```", "") + val todos = objectMapper.readValue>("[$result]") + + val defaultObjectMapper = jacksonObjectMapper() + + val assistantTodoEntity = AssistantTodoEntity( + current.email, + if (assistantRequest.isGroup) null else assistantRequest.chatId, + if (assistantRequest.isGroup) assistantRequest.chatId else null, + objectMapper.writeValueAsString(participants), + getRecipient(current, assistantRequest), + LocalDateTime.now(), + 1, + assistantRequest.startTime, + assistantRequest.endTime, + objectMapper.writeValueAsString(todos) + ) + todoRepository.save(assistantTodoEntity) + + return Todos( + if (assistantRequest.isGroup) null else assistantRequest.chatId, + if (assistantRequest.isGroup) assistantRequest.chatId else null, + participants, + getRecipient(current, assistantRequest), + LocalDateTime.now(), + 1, + assistantRequest.startTime, + assistantRequest.endTime, + todos.ifEmpty { emptyList() } + ) + } + + override fun getAllNotes(getAllNotesRequest: GetAllNotesRequest): List { + val current = userAuthService.current() + val notes: List = if (getAllNotesRequest.isGroup) { + noteRepository.findAllByAuthorAndGroupChatId(current.email!!, getAllNotesRequest.chatId) + } else { + noteRepository.findAllByAuthorAndChatId(current.email!!, getAllNotesRequest.chatId) + } + + return notes.map { convertFromEntity(it) } + } + + override fun getAllTodos(getAllTodosRequest: GetAllTodosRequest): List { + val current = userAuthService.current() + val todos: List = if (getAllTodosRequest.isGroup) { + todoRepository.findAllByAuthorAndGroupChatId(current.email!!, getAllTodosRequest.chatId) + } else { + todoRepository.findAllByAuthorAndChatId(current.email!!, getAllTodosRequest.chatId) + } + + return todos.map { convertFromEntity(it) } + } + + override fun getAllReminders(getAllRemindersRequest: GetAllRemindersRequest): List { + val current = userAuthService.current() + val reminders: List = if (getAllRemindersRequest.isGroup) { + reminderRepository.findAllByAuthorAndGroupChatId(current.email!!, getAllRemindersRequest.chatId) + } else { + reminderRepository.findAllByAuthorAndChatId(current.email!!, getAllRemindersRequest.chatId) + } + + return reminders.map { convertFromEntity(it) } + } + + private fun getConversation(assistantRequest: AssistantRequest): String { + val messages = loadMessageHistory(assistantRequest) + val conversation = messages.map { message: Message -> message.sender + ": " + message.body } + .joinToString(separator = ";") + return conversation + } + + private fun getParticipants(assistantRequest: AssistantRequest): List? { + val participants: List? = if (assistantRequest.isGroup) { + groupChatService.get(assistantRequest.chatId)?.groupParticipants?.map { it.participant!! } + } else privateChatService.getParticipants(assistantRequest.chatId)?.map { it.username!! } + return participants + } + + private fun loadMessageHistory(assistantRequest: AssistantRequest): List { + return if (assistantRequest.isGroup) { + messageJdbcRepository.findByGroupChatIdAndSentAtBetween( + assistantRequest.chatId, + LocalDateTime.from(assistantRequest.startTime), + LocalDateTime.from(assistantRequest.endTime), + Sort.by("sentAt").ascending() + ) + } else messageJdbcRepository.findByPrivateChatIdAndSentAtBetween( + assistantRequest.chatId, + LocalDateTime.from(assistantRequest.startTime), + LocalDateTime.from(assistantRequest.endTime), + Sort.by("sentAt").ascending() + ) + } + + companion object { + const val PROMPT_FOR_REMINDER = "Here is a conversation wrapped on quotes. " + + "Please read and analyze if there are any reminders for participant: %s. " + + "In case there was mentioned anything important to be reminded about or some arrangement between participant " + + "give me only and json output, in case multiple items, result is array, no other text with following format {\\\"remindAt\\\": \\\"exactDate time in ISO " + + "8601\\\", " + + "\\\"description\\\": \\\"description about topic that have a place at remindAt field\\\"}" + const val PROMPT_TODOS = "Here is a conversation wrapped on quotes. " + + "Please read and analyze if there are any tasks for participant: %s. " + + "In case there was mentioned anything important to be done or some assignment from someone " + + "give me only and json output, in case multiple items, result is array, no other text with following format " + + "{\\\"dueDate\\\": \\\"due date time for task in ISO 8601\\\", " + + "\\\"description\\\": \\\"description about the task\\\"" + + "\\\"executor\\\": \\\"if participant is executor, put him here otherwise skip this task\\\"" + + "\\\"context\\\": \\\"in which context task was raised\\\"" + + "}. In case if dueDate is unknown and can't be extracted, put there null." + } + + private fun convertFromEntity(entity: AssistantNoteEntity): ConversationNotes { + val objectMapper = jacksonObjectMapper() + val membersList: List = entity.members?.let { objectMapper.readValue(it) } ?: emptyList() + val notesList: List = entity.notes?.let { objectMapper.readValue(it) } ?: emptyList() + + return ConversationNotes( + chatId = entity.chatId, + groupChatId = entity.groupChatId, + members = membersList, + recipient = entity.recipient, + generatedAt = entity.generatedAt, + version = entity.version, + startTime = entity.startTime ?: LocalDateTime.now(), + endTime = entity.endTime ?: LocalDateTime.now(), + notes = notesList + ) + } + + private fun convertFromEntity(entity: AssistantTodoEntity): Todos { + val objectMapper = jacksonObjectMapper() + val membersList: List = entity.members?.let { objectMapper.readValue(it) } ?: emptyList() + val todoList: List = entity.todos?.let { objectMapper.readValue(it) } ?: emptyList() + + return Todos( + chatId = entity.chatId, + groupChatId = entity.groupChatId, + members = membersList, + recipient = entity.recipient, + generatedAt = entity.generatedAt, + version = entity.version, + startTime = entity.startTime ?: LocalDateTime.now(), + endTime = entity.endTime ?: LocalDateTime.now(), + todos = todoList + ) + } + + private fun convertFromEntity(entity: AssistantReminderEntity): Reminder { + val objectMapper = jacksonObjectMapper() + val membersList: List = entity.members?.let { objectMapper.readValue(it) } ?: emptyList() + val reminderList: List = entity.reminders?.let { objectMapper.readValue(it) } ?: emptyList() + + return Reminder( + chatId = entity.chatId, + groupChatId = entity.groupChatId, + members = membersList, + recipient = entity.recipient, + generatedAt = entity.generatedAt, + version = entity.version, + startTime = entity.startTime ?: LocalDateTime.now(), + endTime = entity.endTime ?: LocalDateTime.now(), + reminders = reminderList + ) + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/impl/AttachmentServiceImpl.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/AttachmentServiceImpl.kt similarity index 60% rename from src/main/kotlin/io/openfuture/openmessanger/service/impl/AttachmentServiceImpl.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/impl/AttachmentServiceImpl.kt index f5f14b1..6296b51 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/impl/AttachmentServiceImpl.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/AttachmentServiceImpl.kt @@ -1,14 +1,15 @@ -package io.openfuture.openmessanger.service.impl +package io.openfuture.openmessenger.service.impl import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.model.ObjectMetadata import com.amazonaws.util.IOUtils -import io.openfuture.openmessanger.configuration.AwsConfig -import io.openfuture.openmessanger.repository.AttachmentRepository -import io.openfuture.openmessanger.service.AttachmentService +import io.openfuture.openmessenger.configuration.AwsConfig +import io.openfuture.openmessenger.repository.AttachmentRepository +import io.openfuture.openmessenger.service.AttachmentService import org.springframework.stereotype.Service import org.springframework.web.multipart.MultipartFile import java.io.IOException +import java.io.InputStream @Service class AttachmentServiceImpl( @@ -26,6 +27,11 @@ class AttachmentServiceImpl( attachmentRepository.save(file.originalFilename) } + override fun upload(name: String, fileInputStream: InputStream) { + val data = ObjectMetadata() + amazonS3.putObject(awsConfig.attachmentsBucket, name, fileInputStream, data) + } + override fun uploadAndReturnId(file: MultipartFile): Int { val data = ObjectMetadata() data.contentType = file.contentType @@ -36,9 +42,19 @@ class AttachmentServiceImpl( } @Throws(IOException::class) - override fun download(fileName: String?): ByteArray? { + override fun download(fileName: String?, bucket: String): ByteArray? { + val o = amazonS3.getObject(bucket, fileName) + val s3is = o.objectContent + return IOUtils.toByteArray(s3is) + } + + @Throws(IOException::class) + override fun downloadById(id: Int): ByteArray? { + val attachmentResponses = attachmentRepository.get(listOf(id)) + val fileName = attachmentResponses?.get(0)?.name val o = amazonS3.getObject(awsConfig.attachmentsBucket, fileName) val s3is = o.objectContent return IOUtils.toByteArray(s3is) } + } \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/impl/CognitoUserServiceImpl.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/CognitoUserServiceImpl.kt similarity index 96% rename from src/main/kotlin/io/openfuture/openmessanger/service/impl/CognitoUserServiceImpl.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/impl/CognitoUserServiceImpl.kt index 4e8f592..fb5cbbf 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/impl/CognitoUserServiceImpl.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/CognitoUserServiceImpl.kt @@ -1,16 +1,16 @@ -package io.openfuture.openmessanger.service.impl +package io.openfuture.openmessenger.service.impl import com.amazonaws.services.cognitoidp.AWSCognitoIdentityProvider import com.amazonaws.services.cognitoidp.model.* import com.amazonaws.services.cognitoidp.model.UsernameExistsException -import io.openfuture.openmessanger.configuration.AwsConfig -import io.openfuture.openmessanger.domain.enums.CognitoAttributesEnum -import io.openfuture.openmessanger.exception.* -import io.openfuture.openmessanger.exception.InvalidParameterException -import io.openfuture.openmessanger.exception.InvalidPasswordException -import io.openfuture.openmessanger.exception.UserNotFoundException -import io.openfuture.openmessanger.service.CognitoUserService -import io.openfuture.openmessanger.service.dto.UserSignUpRequest +import io.openfuture.openmessenger.configuration.AwsConfig +import io.openfuture.openmessenger.domain.enums.CognitoAttributesEnum +import io.openfuture.openmessenger.exception.* +import io.openfuture.openmessenger.exception.InvalidParameterException +import io.openfuture.openmessenger.exception.InvalidPasswordException +import io.openfuture.openmessenger.exception.UserNotFoundException +import io.openfuture.openmessenger.service.CognitoUserService +import io.openfuture.openmessenger.service.dto.UserSignUpRequest import lombok.RequiredArgsConstructor import lombok.extern.slf4j.Slf4j import org.apache.logging.log4j.util.Strings diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/impl/GroupChatServiceImpl.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/GroupChatServiceImpl.kt similarity index 85% rename from src/main/kotlin/io/openfuture/openmessanger/service/impl/GroupChatServiceImpl.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/impl/GroupChatServiceImpl.kt index e1c4129..0eb4b8a 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/impl/GroupChatServiceImpl.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/GroupChatServiceImpl.kt @@ -1,16 +1,16 @@ -package io.openfuture.openmessanger.service.impl +package io.openfuture.openmessenger.service.impl -import io.openfuture.openmessanger.repository.GroupChatRepository -import io.openfuture.openmessanger.repository.GroupParticipantRepository -import io.openfuture.openmessanger.repository.MessageRepository -import io.openfuture.openmessanger.repository.entity.GroupChat -import io.openfuture.openmessanger.repository.entity.GroupParticipant -import io.openfuture.openmessanger.repository.entity.MessageContentType -import io.openfuture.openmessanger.repository.entity.MessageEntity -import io.openfuture.openmessanger.service.GroupChatService -import io.openfuture.openmessanger.web.request.group.AddParticipantsRequest -import io.openfuture.openmessanger.web.request.group.CreateGroupRequest -import io.openfuture.openmessanger.web.request.group.RemoveParticipantsRequest +import io.openfuture.openmessenger.repository.GroupChatRepository +import io.openfuture.openmessenger.repository.GroupParticipantRepository +import io.openfuture.openmessenger.repository.MessageRepository +import io.openfuture.openmessenger.repository.entity.GroupChat +import io.openfuture.openmessenger.repository.entity.GroupParticipant +import io.openfuture.openmessenger.repository.entity.MessageContentType +import io.openfuture.openmessenger.repository.entity.MessageEntity +import io.openfuture.openmessenger.service.GroupChatService +import io.openfuture.openmessenger.web.request.group.AddParticipantsRequest +import io.openfuture.openmessenger.web.request.group.CreateGroupRequest +import io.openfuture.openmessenger.web.request.group.RemoveParticipantsRequest import jakarta.persistence.EntityNotFoundException import lombok.RequiredArgsConstructor import org.springframework.stereotype.Service diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/impl/MessageServiceImpl.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/MessageServiceImpl.kt similarity index 84% rename from src/main/kotlin/io/openfuture/openmessanger/service/impl/MessageServiceImpl.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/impl/MessageServiceImpl.kt index cb04d08..1f1a07a 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/impl/MessageServiceImpl.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/MessageServiceImpl.kt @@ -1,17 +1,17 @@ -package io.openfuture.openmessanger.service.impl - -import io.openfuture.openmessanger.assistant.gemini.GeminiService -import io.openfuture.openmessanger.repository.* -import io.openfuture.openmessanger.repository.entity.* -import io.openfuture.openmessanger.service.MessageService -import io.openfuture.openmessanger.service.PrivateChatService -import io.openfuture.openmessanger.service.UserService -import io.openfuture.openmessanger.web.request.GroupMessageRequest -import io.openfuture.openmessanger.web.request.MessageRequest -import io.openfuture.openmessanger.web.request.MessageToAssistantRequest -import io.openfuture.openmessanger.web.response.GroupMessageResponse -import io.openfuture.openmessanger.web.response.LastMessage -import io.openfuture.openmessanger.web.response.MessageResponse +package io.openfuture.openmessenger.service.impl + +import io.openfuture.openmessenger.assistant.gemini.GeminiService +import io.openfuture.openmessenger.repository.* +import io.openfuture.openmessenger.repository.entity.* +import io.openfuture.openmessenger.service.MessageService +import io.openfuture.openmessenger.service.PrivateChatService +import io.openfuture.openmessenger.service.UserService +import io.openfuture.openmessenger.web.request.GroupMessageRequest +import io.openfuture.openmessenger.web.request.MessageRequest +import io.openfuture.openmessenger.web.request.MessageToAssistantRequest +import io.openfuture.openmessenger.web.response.GroupMessageResponse +import io.openfuture.openmessenger.web.response.LastMessage +import io.openfuture.openmessenger.web.response.MessageResponse import lombok.RequiredArgsConstructor import lombok.extern.slf4j.Slf4j import org.slf4j.Logger @@ -60,8 +60,9 @@ class MessageServiceImpl( private fun getPrivateChat(request: MessageRequest): PrivateChat { if (request.sender == request.recipient) { val selfChat = privateChatRepository.findSelfChat(request.sender) - if (selfChat?.isPresent == true) { - return selfChat.get() + + if (selfChat != null) { + return selfChat } val newPrivateChat = privateChatRepository.save(PrivateChat("SELF")) val singleParticipant = ChatParticipant(newPrivateChat.id, request.sender) @@ -70,15 +71,16 @@ class MessageServiceImpl( return newPrivateChat } val privateChat = privateChatRepository.findPrivateChatByParticipants(request.sender, request.recipient) - if (privateChat?.isPresent == true) { - return privateChat.get() + + if (privateChat != null) { + return privateChat } val newPrivateChat = privateChatRepository.save(PrivateChat("DEFAULT")) val sender = ChatParticipant(newPrivateChat.id, request.sender) val recipient = ChatParticipant(newPrivateChat.id, request.recipient) chatParticipantRepository.save(sender) chatParticipantRepository.save(recipient) - newPrivateChat.chatParticipants = java.util.List.of(sender, recipient) + newPrivateChat.chatParticipants = listOf(sender, recipient) return newPrivateChat } @@ -94,7 +96,8 @@ class MessageServiceImpl( privateChat.id ) - request.attachments.forEach { attachment -> messageAttachmentRepository.save(MessageAttachment(attachment, message.id)) } + val savedMessageId = messageRepository.save(message) + request.attachments.forEach { attachment -> messageAttachmentRepository.save(MessageAttachment(attachment, savedMessageId)) } return MessageResponse( message.id, @@ -133,16 +136,16 @@ class MessageServiceImpl( LocalDateTime.now(), privateChat.id ) - messageRepository.save(responseMessage) + val id = messageRepository.save(responseMessage) return MessageResponse( - message.id, - message.sender, - message.recipient!!, - message.body!!, - message.contentType, - message.receivedAt!!, - message.sentAt, - message.privateChatId!!, + id, + responseMessage.sender, + responseMessage.recipient!!, + response!!, + responseMessage.contentType, + responseMessage.receivedAt!!, + responseMessage.sentAt, + responseMessage.privateChatId!!, null ) } @@ -232,8 +235,6 @@ class MessageServiceImpl( } private fun convertToMessageResponse(messageEntities: List?): List? { - - return messageEntities?.map { message: MessageEntity? -> val attachments = messageAttachmentRepository.findAllByMessageId(message!!.id) MessageResponse( @@ -244,8 +245,8 @@ class MessageServiceImpl( message.contentType, message.receivedAt!!, message.sentAt, - message.privateChatId!!, - message.groupChatId!!, + message.privateChatId, + message.groupChatId, attachments.map { messageAttachment -> messageAttachment.attachmentId!! } ) } diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/impl/PrivateChatServiceImpl.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/PrivateChatServiceImpl.kt similarity index 63% rename from src/main/kotlin/io/openfuture/openmessanger/service/impl/PrivateChatServiceImpl.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/impl/PrivateChatServiceImpl.kt index 5b1a9e6..e122049 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/impl/PrivateChatServiceImpl.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/PrivateChatServiceImpl.kt @@ -1,8 +1,8 @@ -package io.openfuture.openmessanger.service.impl +package io.openfuture.openmessenger.service.impl -import io.openfuture.openmessanger.repository.ChatParticipantRepository -import io.openfuture.openmessanger.repository.entity.ChatParticipant -import io.openfuture.openmessanger.service.PrivateChatService +import io.openfuture.openmessenger.repository.ChatParticipantRepository +import io.openfuture.openmessenger.repository.entity.ChatParticipant +import io.openfuture.openmessenger.service.PrivateChatService import lombok.RequiredArgsConstructor import org.springframework.stereotype.Service @@ -14,7 +14,7 @@ class PrivateChatServiceImpl( override fun getOtherUser(username: String, chatId: Int?): ChatParticipant? { val participants = chatParticipantRepository.findAllByChatId(chatId) - if (participants!!.size == 1) { + if (participants.size == 1) { return participants[0] } val recipient = participants.stream() @@ -22,4 +22,9 @@ class PrivateChatServiceImpl( .findFirst() return recipient.orElseThrow { IllegalStateException("No other participants of chat") } } + + override fun getParticipants(chatId: Int?): List? { + return chatParticipantRepository.findAllByChatId(chatId) + } + } \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/impl/RecordingManagementServiceImpl.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/RecordingManagementServiceImpl.kt new file mode 100644 index 0000000..e0235fd --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/RecordingManagementServiceImpl.kt @@ -0,0 +1,59 @@ +package io.openfuture.openmessenger.service.impl + +import com.amazonaws.services.s3.AmazonS3 +import com.amazonaws.services.s3.model.ListObjectsV2Request +import com.amazonaws.services.s3.model.ListObjectsV2Result +import com.amazonaws.services.s3.model.ObjectMetadata +import com.amazonaws.services.s3.model.S3Object +import io.openfuture.openmessenger.configuration.AwsConfig +import io.openfuture.openmessenger.kurento.recording.UserSession +import io.openfuture.openmessenger.repository.AttachmentRepository +import io.openfuture.openmessenger.service.RecordingManagementService +import org.slf4j.Logger +import org.slf4j.LoggerFactory.getLogger +import org.springframework.stereotype.Service +import java.io.BufferedInputStream +import java.io.File +import java.io.FileInputStream +import java.io.InputStream +import java.net.URLConnection + + +@Service +class RecordingManagementServiceImpl( + val amazonS3: AmazonS3, + val awsConfig: AwsConfig, + val attachmentRepository: AttachmentRepository +): RecordingManagementService { + override fun list() { + val listObjectsV2Request = ListObjectsV2Request().withBucketName(awsConfig.recordingsBucket) + val listObjectsV2Response: ListObjectsV2Result = amazonS3.listObjectsV2(listObjectsV2Request) + println("Number of objects in the bucket: " + listObjectsV2Response.objectSummaries.size) + listObjectsV2Response.objectSummaries.forEach { + println("file = ${it.key}, size: ${it.lastModified}") + } + } + + override fun uploadToS3(fileFullPath: String): Int { + val fileUri = fileFullPath.removePrefix("file://") + log.info("Uploading file to S3: $fileUri") + val file = File(fileUri) + if (!file.exists()) { + log.warn("File does not exist: $fileUri") + return -1 + } + val data = ObjectMetadata() + val fis: InputStream = BufferedInputStream(FileInputStream(file)) + val mimeType = URLConnection.guessContentTypeFromStream(fis) + data.contentType = mimeType + data.contentLength = file.length() + amazonS3.putObject(awsConfig.recordingsBucket, file.name, fis, data) + val id: Int? = attachmentRepository.save(file.name) + return id!! + } + + companion object { + private val log: Logger = getLogger(UserSession::class.java) + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/impl/SpeechToTextServiceImpl.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/SpeechToTextServiceImpl.kt new file mode 100644 index 0000000..9d719a0 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/SpeechToTextServiceImpl.kt @@ -0,0 +1,41 @@ +package io.openfuture.openmessenger.service.impl + +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import io.openfuture.openmessenger.configuration.AwsConfig +import io.openfuture.openmessenger.configuration.AwsS3Config +import io.openfuture.openmessenger.repository.AttachmentRepository +import io.openfuture.openmessenger.service.AttachmentService +import io.openfuture.openmessenger.service.SpeechToTextService +import io.openfuture.openmessenger.service.TranscribeService +import io.openfuture.openmessenger.service.dto.transcript.TranscriptionResponse +import org.springframework.stereotype.Service + +@Service +class SpeechToTextServiceImpl( + val transcribeService: TranscribeService, + val attachmentService: AttachmentService, + val attachmentRepository: AttachmentRepository, + val awsConfig: AwsConfig +) : SpeechToTextService { + + override fun extractTranscript(attachmentId: Int): String { + val objectMapper = jacksonObjectMapper() + val attachmentResponses = attachmentRepository.get(listOf(attachmentId)) + val fileName: String = attachmentResponses?.get(0)?.name!! + val resultFile = transcribeService.extractText(fileName) + val last = resultFile?.split("/")?.last() + + val download = attachmentService.download(last, awsConfig.transcriptsBucket!!) + val result = objectMapper.readValue(download!!) + return result.results.transcripts.first().transcript + } + + override fun getTranscript(s3FileName: String): String { + val objectMapper = jacksonObjectMapper() + val download = attachmentService.download(s3FileName, awsConfig.transcriptsBucket!!) + val result = objectMapper.readValue(download!!) + return result.results.transcripts.first().transcript + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/impl/UserAuthServiceImpl.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/UserAuthServiceImpl.kt similarity index 85% rename from src/main/kotlin/io/openfuture/openmessanger/service/impl/UserAuthServiceImpl.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/impl/UserAuthServiceImpl.kt index 87368e1..84c94c7 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/impl/UserAuthServiceImpl.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/UserAuthServiceImpl.kt @@ -1,17 +1,17 @@ -package io.openfuture.openmessanger.service.impl +package io.openfuture.openmessenger.service.impl import com.amazonaws.services.cognitoidp.model.AdminListUserAuthEventsResult import com.amazonaws.services.cognitoidp.model.AttributeType import com.amazonaws.services.cognitoidp.model.ChallengeNameType import com.amazonaws.services.cognitoidp.model.ForgotPasswordResult -import io.openfuture.openmessanger.exception.UserNotFoundException -import io.openfuture.openmessanger.repository.UserJpaRepository -import io.openfuture.openmessanger.repository.entity.User -import io.openfuture.openmessanger.service.CognitoUserService -import io.openfuture.openmessanger.service.UserAuthService -import io.openfuture.openmessanger.service.dto.* -import io.openfuture.openmessanger.service.response.UserResponse -import io.openfuture.openmessanger.web.response.AuthenticatedResponse +import io.openfuture.openmessenger.exception.UserNotFoundException +import io.openfuture.openmessenger.repository.UserJpaRepository +import io.openfuture.openmessenger.repository.entity.User +import io.openfuture.openmessenger.service.CognitoUserService +import io.openfuture.openmessenger.service.UserAuthService +import io.openfuture.openmessenger.service.dto.* +import io.openfuture.openmessenger.service.response.UserResponse +import io.openfuture.openmessenger.web.response.AuthenticatedResponse import lombok.extern.slf4j.Slf4j import org.springframework.security.core.context.SecurityContextHolder import org.springframework.stereotype.Service @@ -78,6 +78,10 @@ class UserAuthServiceImpl( override fun createUser(signUpDTO: UserSignUpRequest) { cognitoUserService.signUp(signUpDTO) val user = User(signUpDTO.email, signUpDTO.firstName, signUpDTO.lastName) + user.email = signUpDTO.email + user.firstName = signUpDTO.firstName + user.lastName = signUpDTO.lastName + println("request: $signUpDTO and user: $user") userJpaRepository.save(user) } diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/impl/UserServiceImpl.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/UserServiceImpl.kt similarity index 67% rename from src/main/kotlin/io/openfuture/openmessanger/service/impl/UserServiceImpl.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/impl/UserServiceImpl.kt index 9d7f65e..5e110f3 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/impl/UserServiceImpl.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/UserServiceImpl.kt @@ -1,14 +1,14 @@ -package io.openfuture.openmessanger.service.impl +package io.openfuture.openmessenger.service.impl -import io.openfuture.openmessanger.repository.MessageRepository -import io.openfuture.openmessanger.repository.UserJpaRepository -import io.openfuture.openmessanger.repository.entity.GroupChat -import io.openfuture.openmessanger.repository.entity.User -import io.openfuture.openmessanger.service.GroupChatService -import io.openfuture.openmessanger.service.UserService -import io.openfuture.openmessanger.web.request.user.UserDetailsRequest -import io.openfuture.openmessanger.web.response.GroupInfo -import io.openfuture.openmessanger.web.response.UserDetailsResponse +import io.openfuture.openmessenger.repository.MessageRepository +import io.openfuture.openmessenger.repository.UserJpaRepository +import io.openfuture.openmessenger.repository.entity.GroupChat +import io.openfuture.openmessenger.repository.entity.User +import io.openfuture.openmessenger.service.GroupChatService +import io.openfuture.openmessenger.service.UserService +import io.openfuture.openmessenger.web.request.user.UserDetailsRequest +import io.openfuture.openmessenger.web.response.GroupInfo +import io.openfuture.openmessenger.web.response.UserDetailsResponse import lombok.RequiredArgsConstructor import org.springframework.stereotype.Service diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/impl/VideoServiceImpl.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/VideoServiceImpl.kt new file mode 100644 index 0000000..73c7bd3 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/VideoServiceImpl.kt @@ -0,0 +1,26 @@ +package io.openfuture.openmessenger.service.impl + +import com.xuggle.mediatool.IMediaReader +import com.xuggle.mediatool.ToolFactory +import io.openfuture.openmessenger.service.VideoService +import org.springframework.stereotype.Service + + +@Service +class VideoServiceImpl: VideoService { + + override fun convertToAudio() { + val inputFilename = "videofiles/meeting.mp4" + val outputFilename = "audiofiles/meeting.mp3" + + val mediaReader: IMediaReader = ToolFactory.makeReader(inputFilename) + + mediaReader.addListener(XuggleAudioListener(outputFilename)) + + while (mediaReader.readPacket() == null) { + + } + + } + +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/impl/XuggleAudioListener.kt b/src/main/kotlin/io/openfuture/openmessenger/service/impl/XuggleAudioListener.kt new file mode 100644 index 0000000..b9e7a83 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/impl/XuggleAudioListener.kt @@ -0,0 +1,41 @@ +package io.openfuture.openmessenger.service.impl + +import com.xuggle.mediatool.IMediaListener +import com.xuggle.mediatool.event.* +import java.io.File +import java.io.FileOutputStream +import java.io.IOException + +class XuggleAudioListener(outputFilename: String) : IMediaListener { + + private var outputStream: FileOutputStream? = null + + init { + try { + outputStream = FileOutputStream(File(outputFilename)) + } catch (e: IOException) { + e.printStackTrace() + } + } + + override fun onAudioSamples(event: IAudioSamplesEvent) { + val samples = event.audioSamples + try { + outputStream!!.write(samples.data.getByteArray(0, samples.size)) + } catch (e: IOException) { + e.printStackTrace() + } + } + + override fun onOpen(p0: IOpenEvent?) {} + override fun onClose(p0: ICloseEvent?) {} + override fun onAddStream(event: IAddStreamEvent) {} + override fun onCloseCoder(event: ICloseCoderEvent) {} + override fun onFlush(event: IFlushEvent) {} + override fun onWriteTrailer(p0: IWriteTrailerEvent?) {} + override fun onOpenCoder(event: IOpenCoderEvent) {} + override fun onReadPacket(event: IReadPacketEvent) {} + override fun onWritePacket(p0: IWritePacketEvent?) {} + override fun onWriteHeader(p0: IWriteHeaderEvent?) {} + override fun onVideoPicture(event: IVideoPictureEvent) {} +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessenger/service/job/TranscriptionJobResult.kt b/src/main/kotlin/io/openfuture/openmessenger/service/job/TranscriptionJobResult.kt new file mode 100644 index 0000000..4998eb6 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/service/job/TranscriptionJobResult.kt @@ -0,0 +1,36 @@ +package io.openfuture.openmessenger.service.job + +import com.amazonaws.services.transcribe.AmazonTranscribe +import com.amazonaws.services.transcribe.model.GetTranscriptionJobRequest +import com.amazonaws.services.transcribe.model.GetTranscriptionJobResult +import java.util.concurrent.TimeUnit + +class TranscriptionJobResult( + val transcriptionJobName: String, + val amazonTranscribe: AmazonTranscribe +) : Runnable { + + override fun run() { + var getJobRequest: GetTranscriptionJobRequest = GetTranscriptionJobRequest() + .withTranscriptionJobName(transcriptionJobName) + + var getJobResponse: GetTranscriptionJobResult = amazonTranscribe.getTranscriptionJob(getJobRequest) + + var status = getJobResponse.transcriptionJob.transcriptionJobStatus + + while (!status.equals("COMPLETED") || !status.equals("FAILED")) { + getJobRequest = GetTranscriptionJobRequest() + .withTranscriptionJobName(transcriptionJobName) + getJobResponse = amazonTranscribe.getTranscriptionJob(getJobRequest) + status = getJobResponse.transcriptionJob.transcriptionJobStatus + } + + try { + TimeUnit.SECONDS.sleep(5) + } catch (e: InterruptedException) { + e.printStackTrace() + } + } + +} + diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/response/LoginResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/service/response/LoginResponse.kt similarity index 70% rename from src/main/kotlin/io/openfuture/openmessanger/service/response/LoginResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/response/LoginResponse.kt index 07ca2a5..1373737 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/response/LoginResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/response/LoginResponse.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.service.response +package io.openfuture.openmessenger.service.response data class LoginResponse( var token: String? = null, diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/response/SignUpResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/service/response/SignUpResponse.kt similarity index 79% rename from src/main/kotlin/io/openfuture/openmessanger/service/response/SignUpResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/response/SignUpResponse.kt index 3f9991a..6a66a9b 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/response/SignUpResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/response/SignUpResponse.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.service.response +package io.openfuture.openmessenger.service.response data class SignUpResponse( var message: String? = null, diff --git a/src/main/kotlin/io/openfuture/openmessanger/service/response/UserResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/service/response/UserResponse.kt similarity index 82% rename from src/main/kotlin/io/openfuture/openmessanger/service/response/UserResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/service/response/UserResponse.kt index 9e3122f..2f0144f 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/service/response/UserResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/service/response/UserResponse.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.service.response +package io.openfuture.openmessenger.service.response data class UserResponse( var id: String? = null, diff --git a/src/main/kotlin/io/openfuture/openmessenger/web/HomeController.kt b/src/main/kotlin/io/openfuture/openmessenger/web/HomeController.kt new file mode 100644 index 0000000..9556ce3 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/web/HomeController.kt @@ -0,0 +1,14 @@ +package io.openfuture.openmessenger.web + +import org.springframework.stereotype.Controller +import org.springframework.web.bind.annotation.GetMapping + +@Controller +class HomeController { + + @GetMapping("/home") + fun home(): String { + return "index" + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/controller/AttachmentController.kt b/src/main/kotlin/io/openfuture/openmessenger/web/controller/AttachmentController.kt similarity index 79% rename from src/main/kotlin/io/openfuture/openmessanger/web/controller/AttachmentController.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/controller/AttachmentController.kt index dce7191..ac3768f 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/controller/AttachmentController.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/controller/AttachmentController.kt @@ -1,6 +1,6 @@ -package io.openfuture.openmessanger.web.controller +package io.openfuture.openmessenger.web.controller -import io.openfuture.openmessanger.service.AttachmentService +import io.openfuture.openmessenger.service.AttachmentService import jakarta.servlet.http.HttpServletResponse import lombok.RequiredArgsConstructor import org.springframework.http.HttpHeaders @@ -29,13 +29,13 @@ class AttachmentController ( return attachmentService.uploadAndReturnId(file) } - @GetMapping(value = ["/{fileName}"], produces = [MediaType.APPLICATION_OCTET_STREAM_VALUE]) + @GetMapping(value = ["/download/{id}"], produces = [MediaType.APPLICATION_OCTET_STREAM_VALUE]) @Throws(IOException::class) - fun download(response: HttpServletResponse, @PathVariable(value = "fileName") fileName: String) { - val fileData = attachmentService.download(fileName) + fun download(response: HttpServletResponse, @PathVariable(value = "id") id: Int) { + val fileData = attachmentService.downloadById(id) response.reset() response.contentType = MediaType.APPLICATION_OCTET_STREAM_VALUE - response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment$fileName") + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment$id") response.setContentLength(fileData!!.size) response.outputStream.write(fileData) response.flushBuffer() diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/controller/AuthController.kt b/src/main/kotlin/io/openfuture/openmessenger/web/controller/AuthController.kt similarity index 80% rename from src/main/kotlin/io/openfuture/openmessanger/web/controller/AuthController.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/controller/AuthController.kt index a762a00..18a940a 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/controller/AuthController.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/controller/AuthController.kt @@ -1,16 +1,16 @@ -package io.openfuture.openmessanger.web.controller +package io.openfuture.openmessenger.web.controller -import io.openfuture.openmessanger.service.UserAuthService -import io.openfuture.openmessanger.service.dto.LoginRequest -import io.openfuture.openmessanger.service.dto.LoginSmsVerifyRequest -import io.openfuture.openmessanger.service.dto.RefreshTokenRequest -import io.openfuture.openmessanger.service.dto.UserSignUpRequest -import io.openfuture.openmessanger.service.response.Data -import io.openfuture.openmessanger.service.response.LoginResponse -import io.openfuture.openmessanger.service.response.SignUpResponse -import io.openfuture.openmessanger.service.response.UserResponse -import io.openfuture.openmessanger.web.response.AuthenticatedResponse -import io.openfuture.openmessanger.web.response.BaseResponse +import io.openfuture.openmessenger.service.UserAuthService +import io.openfuture.openmessenger.service.dto.LoginRequest +import io.openfuture.openmessenger.service.dto.LoginSmsVerifyRequest +import io.openfuture.openmessenger.service.dto.RefreshTokenRequest +import io.openfuture.openmessenger.service.dto.UserSignUpRequest +import io.openfuture.openmessenger.service.response.Data +import io.openfuture.openmessenger.service.response.LoginResponse +import io.openfuture.openmessenger.service.response.SignUpResponse +import io.openfuture.openmessenger.service.response.UserResponse +import io.openfuture.openmessenger.web.response.AuthenticatedResponse +import io.openfuture.openmessenger.web.response.BaseResponse import org.springframework.http.HttpStatus import org.springframework.http.ResponseEntity import org.springframework.security.core.annotation.AuthenticationPrincipal @@ -57,11 +57,6 @@ class AuthController(val userAuthService: UserAuthService) { ) } - @GetMapping("/current") - fun current(@AuthenticationPrincipal username: String?): ResponseEntity { - return ResponseEntity(username, HttpStatus.OK) - } - @GetMapping("/user") fun userDetails(): UserResponse? { return userAuthService.current() diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/controller/ChatController.kt b/src/main/kotlin/io/openfuture/openmessenger/web/controller/ChatController.kt similarity index 70% rename from src/main/kotlin/io/openfuture/openmessanger/web/controller/ChatController.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/controller/ChatController.kt index 0d1a8f5..a66ab6a 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/controller/ChatController.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/controller/ChatController.kt @@ -1,7 +1,7 @@ -package io.openfuture.openmessanger.web.controller +package io.openfuture.openmessenger.web.controller -import io.openfuture.openmessanger.service.MessageService -import io.openfuture.openmessanger.web.request.MessageRequest +import io.openfuture.openmessenger.service.MessageService +import io.openfuture.openmessenger.web.request.MessageRequest import org.springframework.messaging.handler.annotation.MessageMapping import org.springframework.messaging.handler.annotation.Payload import org.springframework.stereotype.Controller diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/controller/GroupChatController.kt b/src/main/kotlin/io/openfuture/openmessenger/web/controller/GroupChatController.kt similarity index 78% rename from src/main/kotlin/io/openfuture/openmessanger/web/controller/GroupChatController.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/controller/GroupChatController.kt index e94daa6..c921923 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/controller/GroupChatController.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/controller/GroupChatController.kt @@ -1,13 +1,13 @@ -package io.openfuture.openmessanger.web.controller +package io.openfuture.openmessenger.web.controller -import io.openfuture.openmessanger.repository.GroupParticipantRepository -import io.openfuture.openmessanger.repository.entity.GroupChat -import io.openfuture.openmessanger.repository.entity.GroupParticipant -import io.openfuture.openmessanger.service.GroupChatService -import io.openfuture.openmessanger.web.request.group.AddParticipantsRequest -import io.openfuture.openmessanger.web.request.group.CreateGroupRequest -import io.openfuture.openmessanger.web.request.group.RemoveParticipantsRequest -import io.openfuture.openmessanger.web.response.GroupDetailResponse +import io.openfuture.openmessenger.repository.GroupParticipantRepository +import io.openfuture.openmessenger.repository.entity.GroupChat +import io.openfuture.openmessenger.repository.entity.GroupParticipant +import io.openfuture.openmessenger.service.GroupChatService +import io.openfuture.openmessenger.web.request.group.AddParticipantsRequest +import io.openfuture.openmessenger.web.request.group.CreateGroupRequest +import io.openfuture.openmessenger.web.request.group.RemoveParticipantsRequest +import io.openfuture.openmessenger.web.response.GroupDetailResponse import org.springframework.web.bind.annotation.* @RequestMapping("/api/v1/groups") diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/controller/MessageController.kt b/src/main/kotlin/io/openfuture/openmessenger/web/controller/MessageController.kt similarity index 67% rename from src/main/kotlin/io/openfuture/openmessanger/web/controller/MessageController.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/controller/MessageController.kt index 20a0be7..9e39715 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/controller/MessageController.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/controller/MessageController.kt @@ -1,18 +1,20 @@ -package io.openfuture.openmessanger.web.controller - -import io.openfuture.openmessanger.service.MessageService -import io.openfuture.openmessanger.web.request.GroupMessageRequest -import io.openfuture.openmessanger.web.request.MessageRequest -import io.openfuture.openmessanger.web.request.MessageToAssistantRequest -import io.openfuture.openmessanger.web.response.GroupMessageResponse -import io.openfuture.openmessanger.web.response.LastMessage -import io.openfuture.openmessanger.web.response.MessageResponse +package io.openfuture.openmessenger.web.controller + +import io.openfuture.openmessenger.service.MessageService +import io.openfuture.openmessenger.service.UserAuthService +import io.openfuture.openmessenger.web.request.GroupMessageRequest +import io.openfuture.openmessenger.web.request.MessageRequest +import io.openfuture.openmessenger.web.request.MessageToAssistantRequest +import io.openfuture.openmessenger.web.response.GroupMessageResponse +import io.openfuture.openmessenger.web.response.LastMessage +import io.openfuture.openmessenger.web.response.MessageResponse import org.springframework.web.bind.annotation.* @RequestMapping("/api/v1/messages") @RestController class MessageController( - val messageService: MessageService + val messageService: MessageService, + val userAuthService: UserAuthService ) { @GetMapping(value = ["/recipient/{username}"]) @@ -44,8 +46,9 @@ class MessageController( } @GetMapping(value = ["/front-messages"]) - fun getFrontMessages(@RequestParam(value = "user") username: String): List? { - return messageService.getLastMessagesByRecipient(username) + fun getFrontMessages(): List? { + val current: String? = userAuthService.current().email + return messageService.getLastMessagesByRecipient(current!!) } @GetMapping(value = ["/chat/{chatId}"]) diff --git a/src/main/kotlin/io/openfuture/openmessenger/web/controller/PromptController.kt b/src/main/kotlin/io/openfuture/openmessenger/web/controller/PromptController.kt new file mode 100644 index 0000000..7033af9 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/web/controller/PromptController.kt @@ -0,0 +1,58 @@ +package io.openfuture.openmessenger.web.controller + +import io.openfuture.openmessenger.assistant.model.ConversationNotes +import io.openfuture.openmessenger.assistant.model.Reminder +import io.openfuture.openmessenger.assistant.model.Todos +import io.openfuture.openmessenger.service.AssistantService +import io.openfuture.openmessenger.service.dto.AssistantRequest +import io.openfuture.openmessenger.service.dto.GetAllNotesRequest +import io.openfuture.openmessenger.service.dto.GetAllRemindersRequest +import io.openfuture.openmessenger.service.dto.GetAllTodosRequest +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RequestMapping("/api/v1/ai") +@RestController +class PromptController( + val assistantService: AssistantService +) { + + @PostMapping("/generateNotes") + fun generateNotes( + @RequestBody request: AssistantRequest + ): ConversationNotes? { + return assistantService.generateNotes(request) + } + + @PostMapping("/generateReminders") + fun generateReminders( + @RequestBody request: AssistantRequest + ): Reminder? { + return assistantService.generateReminder(request) + } + + @PostMapping("/generateTodos") + fun generateTodos( + @RequestBody request: AssistantRequest + ): Todos? { + return assistantService.generateTodos(request) + } + + @PostMapping("/notes") + fun getAllNotes(@RequestBody request: GetAllNotesRequest): List { + return assistantService.getAllNotes(request) + } + + @PostMapping("/todos") + fun getAllTodos(@RequestBody request: GetAllTodosRequest): List { + return assistantService.getAllTodos(request) + } + + @PostMapping("/reminders") + fun getAllReminders(@RequestBody request: GetAllRemindersRequest): List { + return assistantService.getAllReminders(request) + } + +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/web/controller/RecordingController.kt b/src/main/kotlin/io/openfuture/openmessenger/web/controller/RecordingController.kt new file mode 100644 index 0000000..0884a74 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/web/controller/RecordingController.kt @@ -0,0 +1,20 @@ +package io.openfuture.openmessenger.web.controller + +import io.openfuture.openmessenger.service.RecordingManagementService +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + + +@RequestMapping("/api/v1/recordings") +@RestController +class RecordingController( + val recordingManagementService: RecordingManagementService +) { + + @GetMapping("/list") + fun generateNotes() { + recordingManagementService.list() + } + +} diff --git a/src/main/kotlin/io/openfuture/openmessenger/web/controller/SpeechToTextController.kt b/src/main/kotlin/io/openfuture/openmessenger/web/controller/SpeechToTextController.kt new file mode 100644 index 0000000..fa97d64 --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/web/controller/SpeechToTextController.kt @@ -0,0 +1,37 @@ +package io.openfuture.openmessenger.web.controller + +import io.openfuture.openmessenger.assistant.gemini.GeminiService +import io.openfuture.openmessenger.service.SpeechToTextService +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RequestMapping("/api/v1/ai") +@RestController +class SpeechToTextController( + val geminiService: GeminiService, + val speechToTextService: SpeechToTextService +) { + + @GetMapping("/generateSummary/{attachmentId}") + fun generateNotes( + @PathVariable attachmentId: Int + ): String { + val transcript = speechToTextService.extractTranscript(attachmentId) + val chat = geminiService.chat("Generate a summary from the following meeting record: {$transcript}") + + return chat!! + } + + @GetMapping("/extractTranscript/{attachmentId}") + fun extractTranscript(@PathVariable attachmentId: Int): String { + return speechToTextService.extractTranscript(attachmentId) + } + + @GetMapping("/getTranscriptFromS3/{s3FileName}") + fun getTranscript(@PathVariable s3FileName: String): String { + return speechToTextService.getTranscript(s3FileName) + } + +} diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/controller/UserController.kt b/src/main/kotlin/io/openfuture/openmessenger/web/controller/UserController.kt similarity index 77% rename from src/main/kotlin/io/openfuture/openmessanger/web/controller/UserController.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/controller/UserController.kt index 76d86e1..46505c1 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/controller/UserController.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/controller/UserController.kt @@ -1,9 +1,9 @@ -package io.openfuture.openmessanger.web.controller +package io.openfuture.openmessenger.web.controller -import io.openfuture.openmessanger.repository.entity.User -import io.openfuture.openmessanger.service.UserService -import io.openfuture.openmessanger.web.request.user.UserDetailsRequest -import io.openfuture.openmessanger.web.response.UserDetailsResponse +import io.openfuture.openmessenger.repository.entity.User +import io.openfuture.openmessenger.service.UserService +import io.openfuture.openmessenger.web.request.user.UserDetailsRequest +import io.openfuture.openmessenger.web.response.UserDetailsResponse import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.security.core.annotation.AuthenticationPrincipal diff --git a/src/main/kotlin/io/openfuture/openmessenger/web/controller/UtilsController.kt b/src/main/kotlin/io/openfuture/openmessenger/web/controller/UtilsController.kt new file mode 100644 index 0000000..967bf0d --- /dev/null +++ b/src/main/kotlin/io/openfuture/openmessenger/web/controller/UtilsController.kt @@ -0,0 +1,24 @@ +package io.openfuture.openmessenger.web.controller + +import io.openfuture.openmessenger.service.AttachmentService +import io.openfuture.openmessenger.service.VideoService +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController +import java.io.File +import java.io.FileInputStream + +@RestController +@RequestMapping("/api/v1/utils") +class UtilsController( + val videoService: VideoService, + val attachmentService: AttachmentService +) { + + @PostMapping("extractAudio") + fun sparks() { + val fileInputStream = FileInputStream(File("audiofiles/meeting.mp3")) + attachmentService.upload("meeting-11-07-2024.mp3", fileInputStream) + } + +} \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/request/GroupMessageRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/web/request/GroupMessageRequest.kt similarity index 67% rename from src/main/kotlin/io/openfuture/openmessanger/web/request/GroupMessageRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/request/GroupMessageRequest.kt index 9c859bf..45ad7af 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/request/GroupMessageRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/request/GroupMessageRequest.kt @@ -1,6 +1,6 @@ -package io.openfuture.openmessanger.web.request +package io.openfuture.openmessenger.web.request -import io.openfuture.openmessanger.repository.entity.MessageContentType +import io.openfuture.openmessenger.repository.entity.MessageContentType data class GroupMessageRequest( val sender: String? = null, diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/request/MessageRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/web/request/MessageRequest.kt similarity index 53% rename from src/main/kotlin/io/openfuture/openmessanger/web/request/MessageRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/request/MessageRequest.kt index e64dca1..e121441 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/request/MessageRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/request/MessageRequest.kt @@ -1,9 +1,6 @@ -package io.openfuture.openmessanger.web.request +package io.openfuture.openmessenger.web.request -import io.openfuture.openmessanger.repository.entity.MessageContentType -import lombok.AllArgsConstructor -import lombok.Data -import lombok.NoArgsConstructor +import io.openfuture.openmessenger.repository.entity.MessageContentType data class MessageRequest( val sender: String? = null, diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/request/MessageToAssistantRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/web/request/MessageToAssistantRequest.kt similarity index 58% rename from src/main/kotlin/io/openfuture/openmessanger/web/request/MessageToAssistantRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/request/MessageToAssistantRequest.kt index 214a1bb..a349508 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/request/MessageToAssistantRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/request/MessageToAssistantRequest.kt @@ -1,6 +1,6 @@ -package io.openfuture.openmessanger.web.request +package io.openfuture.openmessenger.web.request -import io.openfuture.openmessanger.repository.entity.MessageContentType +import io.openfuture.openmessenger.repository.entity.MessageContentType data class MessageToAssistantRequest( val sender: String? = null, diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/request/group/AddParticipantsRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/web/request/group/AddParticipantsRequest.kt similarity index 65% rename from src/main/kotlin/io/openfuture/openmessanger/web/request/group/AddParticipantsRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/request/group/AddParticipantsRequest.kt index ed534e9..1ce361d 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/request/group/AddParticipantsRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/request/group/AddParticipantsRequest.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.web.request.group +package io.openfuture.openmessenger.web.request.group data class AddParticipantsRequest( var groupId: Int? = null, diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/request/group/CommonGroupsRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/web/request/group/CommonGroupsRequest.kt similarity index 64% rename from src/main/kotlin/io/openfuture/openmessanger/web/request/group/CommonGroupsRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/request/group/CommonGroupsRequest.kt index 250842f..a9b72ce 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/request/group/CommonGroupsRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/request/group/CommonGroupsRequest.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.web.request.group +package io.openfuture.openmessenger.web.request.group data class CommonGroupsRequest( var username: String? = null, diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/request/group/CreateGroupRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/web/request/group/CreateGroupRequest.kt similarity index 72% rename from src/main/kotlin/io/openfuture/openmessanger/web/request/group/CreateGroupRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/request/group/CreateGroupRequest.kt index 1ff8bf6..a9e1cef 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/request/group/CreateGroupRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/request/group/CreateGroupRequest.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.web.request.group +package io.openfuture.openmessenger.web.request.group data class CreateGroupRequest( var name: String? = null, diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/request/group/RemoveParticipantsRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/web/request/group/RemoveParticipantsRequest.kt similarity index 66% rename from src/main/kotlin/io/openfuture/openmessanger/web/request/group/RemoveParticipantsRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/request/group/RemoveParticipantsRequest.kt index 5d0169b..10b22b0 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/request/group/RemoveParticipantsRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/request/group/RemoveParticipantsRequest.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.web.request.group +package io.openfuture.openmessenger.web.request.group data class RemoveParticipantsRequest( var groupId: Int? = null, diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/request/user/UserDetailsRequest.kt b/src/main/kotlin/io/openfuture/openmessenger/web/request/user/UserDetailsRequest.kt similarity index 57% rename from src/main/kotlin/io/openfuture/openmessanger/web/request/user/UserDetailsRequest.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/request/user/UserDetailsRequest.kt index 0ed085f..2bf46ba 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/request/user/UserDetailsRequest.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/request/user/UserDetailsRequest.kt @@ -1,3 +1,3 @@ -package io.openfuture.openmessanger.web.request.user +package io.openfuture.openmessenger.web.request.user data class UserDetailsRequest(val username: String, val email: String) \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/response/AuthenticatedResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/web/response/AuthenticatedResponse.kt similarity index 81% rename from src/main/kotlin/io/openfuture/openmessanger/web/response/AuthenticatedResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/response/AuthenticatedResponse.kt index 53df1ce..125f8f0 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/response/AuthenticatedResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/response/AuthenticatedResponse.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.web.response +package io.openfuture.openmessenger.web.response import java.io.Serializable diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/response/BaseResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/web/response/BaseResponse.kt similarity index 64% rename from src/main/kotlin/io/openfuture/openmessanger/web/response/BaseResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/response/BaseResponse.kt index 2eebe49..c57b320 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/response/BaseResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/response/BaseResponse.kt @@ -1,3 +1,3 @@ -package io.openfuture.openmessanger.web.response +package io.openfuture.openmessenger.web.response data class BaseResponse(val data: Any?, val message: String?, val error: Boolean = true) \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/response/CommonGroupsResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/web/response/CommonGroupsResponse.kt similarity index 70% rename from src/main/kotlin/io/openfuture/openmessanger/web/response/CommonGroupsResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/response/CommonGroupsResponse.kt index e1ec790..0efbcff 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/response/CommonGroupsResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/response/CommonGroupsResponse.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.web.response +package io.openfuture.openmessenger.web.response data class CommonGroupsResponse( var email: String, diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/response/FrontMessagesResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/web/response/FrontMessagesResponse.kt similarity index 65% rename from src/main/kotlin/io/openfuture/openmessanger/web/response/FrontMessagesResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/response/FrontMessagesResponse.kt index a660ef1..a7098ac 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/response/FrontMessagesResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/response/FrontMessagesResponse.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.web.response +package io.openfuture.openmessenger.web.response data class FrontMessagesResponse ( var lastMessages: Collection? = null diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/response/GroupDetailResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/web/response/GroupDetailResponse.kt similarity index 80% rename from src/main/kotlin/io/openfuture/openmessanger/web/response/GroupDetailResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/response/GroupDetailResponse.kt index f7a1cbc..5eba123 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/response/GroupDetailResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/response/GroupDetailResponse.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.web.response +package io.openfuture.openmessenger.web.response data class GroupDetailResponse( var id: Int? = null, diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/response/GroupInfo.kt b/src/main/kotlin/io/openfuture/openmessenger/web/response/GroupInfo.kt similarity index 61% rename from src/main/kotlin/io/openfuture/openmessanger/web/response/GroupInfo.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/response/GroupInfo.kt index 57311f8..d0919b0 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/response/GroupInfo.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/response/GroupInfo.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.web.response +package io.openfuture.openmessenger.web.response data class GroupInfo( var id: Int? = null, diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/response/GroupMessageResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/web/response/GroupMessageResponse.kt similarity index 85% rename from src/main/kotlin/io/openfuture/openmessanger/web/response/GroupMessageResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/response/GroupMessageResponse.kt index 3dcd8bf..10944be 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/response/GroupMessageResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/response/GroupMessageResponse.kt @@ -1,6 +1,6 @@ -package io.openfuture.openmessanger.web.response +package io.openfuture.openmessenger.web.response -import io.openfuture.openmessanger.repository.entity.MessageContentType +import io.openfuture.openmessenger.repository.entity.MessageContentType import java.time.LocalDateTime import java.time.ZonedDateTime diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/response/LastMessage.kt b/src/main/kotlin/io/openfuture/openmessenger/web/response/LastMessage.kt similarity index 88% rename from src/main/kotlin/io/openfuture/openmessanger/web/response/LastMessage.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/response/LastMessage.kt index 10fa576..e9400a8 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/response/LastMessage.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/response/LastMessage.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.web.response +package io.openfuture.openmessenger.web.response import java.time.LocalDateTime diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/response/MessageResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/web/response/MessageResponse.kt similarity index 70% rename from src/main/kotlin/io/openfuture/openmessanger/web/response/MessageResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/response/MessageResponse.kt index 73e1482..38a7fa8 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/response/MessageResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/response/MessageResponse.kt @@ -1,6 +1,6 @@ -package io.openfuture.openmessanger.web.response +package io.openfuture.openmessenger.web.response -import io.openfuture.openmessanger.repository.entity.MessageContentType +import io.openfuture.openmessenger.repository.entity.MessageContentType import java.time.LocalDateTime data class MessageResponse( @@ -11,7 +11,7 @@ data class MessageResponse( var contentType: MessageContentType, var receivedAt: LocalDateTime, var sentAt: LocalDateTime, - var privateChatId: Int, + var privateChatId: Int?, var groupChatId: Int?, var attachments: List = emptyList() ) \ No newline at end of file diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/response/UserDetailsResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/web/response/UserDetailsResponse.kt similarity index 74% rename from src/main/kotlin/io/openfuture/openmessanger/web/response/UserDetailsResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/response/UserDetailsResponse.kt index be3771b..dcfc802 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/response/UserDetailsResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/response/UserDetailsResponse.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.web.response +package io.openfuture.openmessenger.web.response data class UserDetailsResponse( var email: String? = null, diff --git a/src/main/kotlin/io/openfuture/openmessanger/web/response/UserMessageResponse.kt b/src/main/kotlin/io/openfuture/openmessenger/web/response/UserMessageResponse.kt similarity index 84% rename from src/main/kotlin/io/openfuture/openmessanger/web/response/UserMessageResponse.kt rename to src/main/kotlin/io/openfuture/openmessenger/web/response/UserMessageResponse.kt index 1357945..e6b19cc 100644 --- a/src/main/kotlin/io/openfuture/openmessanger/web/response/UserMessageResponse.kt +++ b/src/main/kotlin/io/openfuture/openmessenger/web/response/UserMessageResponse.kt @@ -1,4 +1,4 @@ -package io.openfuture.openmessanger.web.response +package io.openfuture.openmessenger.web.response import java.time.LocalDateTime diff --git a/src/main/resources/application-postgres.yml b/src/main/resources/application-postgres.yml deleted file mode 100644 index 249e645..0000000 --- a/src/main/resources/application-postgres.yml +++ /dev/null @@ -1,10 +0,0 @@ -spring: - application: - name: open-chat - datasource: - driver-class-name: org.postgresql.Driver - url: jdbc:postgresql://localhost:5434/open_chat - username: postgres - password: 123456 - jpa: - database-platform: org.hibernate.dialect.PostgreSQLDialect diff --git a/src/main/resources/application-production.yml b/src/main/resources/application-production.yml new file mode 100644 index 0000000..ed48cf6 --- /dev/null +++ b/src/main/resources/application-production.yml @@ -0,0 +1,17 @@ +spring: + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://${POSTGRES_URL} + username: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} + +kms: + url: ${KMS_URL} + +server: + ssl: + key-store: classpath:keystore.p12 + key-store-password: 12345678 + key-store-type: PKCS12 + key-alias: tomcat + port: 8443 diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 03b925c..e6318ff 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -8,13 +8,21 @@ spring: password: 123456 jpa: database-platform: org.hibernate.dialect.PostgreSQLDialect - show-sql: true - + cloud: + gcp: + credentials: + location: file:/Users/beksultan/Documents/open-messanger/alien-scope-428812-i2-0794e36fcbee.json + servlet: + multipart: + max-file-size: 50MB + max-request-size: 50MB aws: - access-key: AKIAXYKJRLFYPXI732OX - secret-key: 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 + access-key: AKIAXYKJRLFYOGZSQSGZ + secret-key: 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 region: us-east-2 - attachments-bucket: open-chat-attachments + attachments-bucket: open-attachments + recordings-bucket: open-recordings + transcripts-bucket: open-transcripts cognito: user-pool-id: us-east-2_cQqrqGoe5 app-client-id: 661d1ashh62bvj8m9g4r2699qm @@ -32,4 +40,19 @@ openai: gemini: api: key: AIzaSyDWkne0n1KzsTZWEzp3x7SaHyafjd0ueHU - url: https://generativelanguage.googleapis.com/v1beta/models \ No newline at end of file + url: https://generativelanguage.googleapis.com/v1beta/models + + +system: + GOOGLE_APPLICATION_CREDENTIALS: /Users/beksultan/Documents/open-messanger/alien-scope-428812-i2-0794e36fcbee.json + +kms: + url: ws://127.0.0.1:8888/kurento + +server: + ssl: + key-store: classpath:keystore.jks + key-store-password: kurento + key-store-type: JKS + key-alias: kurento-selfsigned + port: 8443 diff --git a/src/main/resources/db/migration/V3__create-tables-3.sql b/src/main/resources/db/migration/V3__create-tables-3.sql new file mode 100644 index 0000000..b2b2708 --- /dev/null +++ b/src/main/resources/db/migration/V3__create-tables-3.sql @@ -0,0 +1,44 @@ +create table assistant_notes +( + id serial primary key, + author varchar(255), + chat_id int null, + group_chat_id int null, + members text, + recipient varchar(255), + generated_at timestamp, + version int, + start_time timestamp, + end_time timestamp, + notes text +); + +create table assistant_todos +( + id serial primary key, + author varchar(255), + chat_id int null, + group_chat_id int null, + members text, + recipient varchar(255), + generated_at timestamp, + version int, + start_time timestamp, + end_time timestamp, + todos text +); + +create table assistant_reminders +( + id serial primary key, + author varchar(255), + chat_id int null, + group_chat_id int null, + members text, + recipient varchar(255), + generated_at timestamp, + version int, + start_time timestamp, + end_time timestamp, + reminders text +); \ No newline at end of file diff --git a/src/main/resources/db/migration/V4__create-tables-4.sql b/src/main/resources/db/migration/V4__create-tables-4.sql new file mode 100644 index 0000000..d3c7450 --- /dev/null +++ b/src/main/resources/db/migration/V4__create-tables-4.sql @@ -0,0 +1,14 @@ +create table meeting_notes +( + id serial primary key, + author varchar(255), + chat_id int null, + group_chat_id int null, + members text, + recipient varchar(255), + generated_at timestamp, + version int, + start_time timestamp, + end_time timestamp, + notes text +); diff --git a/src/main/resources/keystore.jks b/src/main/resources/keystore.jks new file mode 100644 index 0000000..c13f803 Binary files /dev/null and b/src/main/resources/keystore.jks differ diff --git a/src/main/resources/keystore.p12 b/src/main/resources/keystore.p12 new file mode 100644 index 0000000..61dc230 Binary files /dev/null and b/src/main/resources/keystore.p12 differ diff --git a/src/main/resources/openfuture b/src/main/resources/openfuture new file mode 100644 index 0000000..957697c Binary files /dev/null and b/src/main/resources/openfuture differ diff --git a/src/main/resources/static/app.js b/src/main/resources/static/app.js deleted file mode 100644 index 9ea5e4f..0000000 --- a/src/main/resources/static/app.js +++ /dev/null @@ -1,65 +0,0 @@ -const stompClient = new StompJs.Client({ - brokerURL: 'ws://localhost:8080/ws' -}); - -stompClient.onConnect = (frame) => { - setConnected(true); - console.log('Connected: ' + frame); - stompClient.subscribe('/user/fire/direct', (greeting) => { - showMessage(JSON.parse(greeting.body).body); - }); -}; - -stompClient.onWebSocketError = (error) => { - console.error('Error with websocket', error); -}; - -stompClient.onStompError = (frame) => { - console.error('Broker reported error: ' + frame.headers['message']); - console.error('Additional details: ' + frame.body); -}; - -function setConnected(connected) { - $("#connect").prop("disabled", connected); - $("#disconnect").prop("disabled", !connected); - if (connected) { - $("#conversation").show(); - } - else { - $("#conversation").hide(); - } - $("#greetings").html(""); -} - -function connect() { - stompClient.activate(); -} - -function disconnect() { - stompClient.deactivate(); - setConnected(false); - console.log("Disconnected"); -} - -function sendName() { - stompClient.publish({ - destination: "/app/direct-message", - body: JSON.stringify( - { - 'sender': 'ice', - 'recipient': 'fire', - 'body': 'randomText' - }) - }); -} - -function showMessage(message) { - $("#greetings").append("" + message + ""); -} - -$(function () { - $("form").on('submit', (e) => e.preventDefault()); - $( "#connect" ).click(() => connect()); - $( "#disconnect" ).click(() => disconnect()); - $( "#send" ).click(() => sendName()); -}); \ No newline at end of file diff --git a/src/main/resources/static/call.html b/src/main/resources/static/call.html new file mode 100644 index 0000000..36448f1 --- /dev/null +++ b/src/main/resources/static/call.html @@ -0,0 +1,26 @@ + + + + + + Video Call Screen + + + +
+ +
+ + +
+ + +
+ + + +
+
+ + + diff --git a/src/main/resources/static/css/call.css b/src/main/resources/static/css/call.css new file mode 100644 index 0000000..d0b47b8 --- /dev/null +++ b/src/main/resources/static/css/call.css @@ -0,0 +1,75 @@ +body { + margin: 0; + font-family: Arial, sans-serif; + background-color: #000; + color: white; + display: flex; + justify-content: center; + align-items: center; + height: 100vh; +} + +.video-call-container { + display: flex; + flex-direction: column; + justify-content: space-between; + width: 100%; + height: 100%; +} + +.video-wrapper { + position: relative; + flex-grow: 1; + display: flex; + justify-content: center; + align-items: center; + overflow: hidden; +} + +video { + max-width: 100%; + max-height: 100%; + border-radius: 10px; + background-color: black; +} + +#uiLocalVideo { + position: absolute; + width: 30%; + height: auto; + top: 10px; + right: 10px; + border: 2px solid white; + border-radius: 8px; +} + +.controls { + display: flex; + justify-content: center; + gap: 15px; + padding: 15px; + background: rgba(0, 0, 0, 0.6); +} + +.control-btn { + background-color: #333; + border: none; + color: white; + padding: 10px 20px; + border-radius: 5px; + font-size: 20px; + cursor: pointer; +} + +.control-btn:hover { + background-color: #555; +} + +.end-call { + background-color: red; + color: white; +} + +.end-call:hover { + background-color: darkred; +} diff --git a/src/main/resources/static/css/kurento.css b/src/main/resources/static/css/kurento.css new file mode 100644 index 0000000..8a01f28 --- /dev/null +++ b/src/main/resources/static/css/kurento.css @@ -0,0 +1,73 @@ +/* + * (C) Copyright 2014 Kurento (https://kurento.openvidu.io/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +@CHARSET "UTF-8"; + +html { + position: relative; + min-height: 100%; +} + +body { + padding-top: 40px; +} + +video, #console { + display: block; + font-size: 14px; + line-height: 1.42857143; + color: #555; + background-color: #fff; + background-image: none; + border: 1px solid #ccc; + border-radius: 4px; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, .075); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, .075); + -webkit-transition: border-color ease-in-out .15s, box-shadow + ease-in-out .15s; + transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s; +} + +#console { + overflow-y: auto; + width: 100%; + height: 120px; +} + +.col-md-2 { + width: 80px; + padding-top: 190px; +} + +.video-container { + height: 480px; + width: 720px; + position:relative; +} +.userVideo { + height: 100%; /* Fill the container */ + width: 100%; /* Fill the container */ + position: relative; +} + +.peerVideo { + height: 200px; + width: 200px; + border: 5px solid yellow; + position: absolute; + bottom: 0; /* Bottom of the .video-container */ + right: 0; +} \ No newline at end of file diff --git a/src/main/resources/static/css/recording-page.css b/src/main/resources/static/css/recording-page.css new file mode 100644 index 0000000..1c84ecd --- /dev/null +++ b/src/main/resources/static/css/recording-page.css @@ -0,0 +1,63 @@ +@CHARSET "UTF-8"; + +html { + position: relative; + min-height: 100%; +} + +body { + padding-top: 40px; +} + +video, #console { + display: block; + font-size: 14px; + line-height: 1.42857143; + color: #555; + background-color: #fff; + background-image: none; + border: 1px solid #ccc; + border-radius: 4px; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, .075); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, .075); + -webkit-transition: border-color ease-in-out .15s, box-shadow + ease-in-out .15s; + transition: border-color ease-in-out .15s, box-shadow ease-in-out .15s; +} + +#console { + overflow-y: auto; + width: 100%; + height: 175px; +} + +#videoContainer { + position: absolute; + float: left; +} + +#videoBig { + width: 640px; + height: 480px; + top: 0; + left: 0; + z-index: 1; +} + +div#videoSmall { + width: 240px; + height: 180px; + padding: 0px; + position: absolute; + top: 15px; + left: 400px; + cursor: pointer; + z-index: 10; + padding: 0px; +} + +div.dragged { + cursor: all-scroll !important; + border-color: blue !important; + z-index: 10 !important; +} diff --git a/src/main/resources/static/css/style.css b/src/main/resources/static/css/style.css new file mode 100644 index 0000000..c19b3ef --- /dev/null +++ b/src/main/resources/static/css/style.css @@ -0,0 +1,449 @@ +/* + * (C) Copyright 2016 Kurento (https://kurento.openvidu.io/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +@CHARSET "UTF-8"; + +body { + font: 13px/20px "Lucida Grande", Tahoma, Verdana, sans-serif; + color: #404040; + background: #0ca3d2; +} + +input[type=checkbox], input[type=radio] { + border: 1px solid #c0c0c0; + margin: 0 0.1em 0 0; + padding: 0; + font-size: 16px; + line-height: 1em; + width: 1.25em; + height: 1.25em; + background: #fff; + background: -webkit-gradient(linear, 0% 0%, 0% 100%, from(#ededed), + to(#fbfbfb)); + -webkit-appearance: none; + -webkit-box-shadow: 1px 1px 1px #fff; + -webkit-border-radius: 0.25em; + vertical-align: text-top; + display: inline-block; +} + +input[type=radio] { + -webkit-border-radius: 2em; /* Make radios round */ +} + +input[type=checkbox]:checked::after { + content: "✔"; + display: block; + text-align: center; + font-size: 16px; + height: 16px; + line-height: 18px; +} + +input[type=radio]:checked::after { + content: "●"; + display: block; + height: 16px; + line-height: 15px; + font-size: 20px; + text-align: center; +} + +select { + border: 1px solid #D0D0D0; + background: url(http://www.quilor.com/i/select.png) no-repeat right + center, -webkit-gradient(linear, 0% 0%, 0% 100%, from(#fbfbfb), + to(#ededed)); + background: -moz-linear-gradient(19% 75% 90deg, #ededed, #fbfbfb); + -webkit-box-shadow: 0 1px 2px rgba(0, 0, 0, .05); + -moz-box-shadow: 0 1px 2px rgba(0, 0, 0, .05); + -webkit-box-shadow: 0 1px 2px rgba(0, 0, 0, .05); + color: #444; +} + +.container { + margin: 50px auto; + width: 640px; +} + +.join { + position: relative; + margin: 0 auto; + padding: 20px 20px 20px; + width: 310px; + background: white; + border-radius: 3px; + -webkit-box-shadow: 0 0 200px rgba(255, 255, 255, 0.5), 0 1px 2px + rgba(0, 0, 0, 0.3); + box-shadow: 0 0 200px rgba(255, 255, 255, 0.5), 0 1px 2px + rgba(0, 0, 0, 0.3); + /*Transition*/ + -webkit-transition: all 0.3s linear; + -moz-transition: all 0.3s linear; + -o-transition: all 0.3s linear; + transition: all 0.3s linear; +} + +.join:before { + content: ''; + position: absolute; + top: -8px; + right: -8px; + bottom: -8px; + left: -8px; + z-index: -1; + background: rgba(0, 0, 0, 0.08); + border-radius: 4px; +} + +.join h1 { + margin: -20px -20px 21px; + line-height: 40px; + font-size: 15px; + font-weight: bold; + color: #555; + text-align: center; + text-shadow: 0 1px white; + background: #f3f3f3; + border-bottom: 1px solid #cfcfcf; + border-radius: 3px 3px 0 0; + background-image: -webkit-linear-gradient(top, whiteffd, #eef2f5); + background-image: -moz-linear-gradient(top, whiteffd, #eef2f5); + background-image: -o-linear-gradient(top, whiteffd, #eef2f5); + background-image: linear-gradient(to bottom, whiteffd, #eef2f5); + -webkit-box-shadow: 0 1px whitesmoke; + box-shadow: 0 1px whitesmoke; +} + +.join p { + margin: 20px 0 0; +} + +.join p:first-child { + margin-top: 0; +} + +.join input[type=text], .join input[type=password] { + width: 278px; +} + +.join p.submit { + text-align: center; +} + +:-moz-placeholder { + color: #c9c9c9 !important; + font-size: 13px; +} + +::-webkit-input-placeholder { + color: #ccc; + font-size: 13px; +} + +input { + font-family: 'Lucida Grande', Tahoma, Verdana, sans-serif; + font-size: 14px; +} + +input[type=text], input[type=password] { + margin: 5px; + padding: 0 10px; + width: 200px; + height: 34px; + color: #404040; + background: white; + border: 1px solid; + border-color: #c4c4c4 #d1d1d1 #d4d4d4; + border-radius: 2px; + outline: 5px solid #eff4f7; + -moz-outline-radius: 3px; + -webkit-box-shadow: inset 0 1px 3px rgba(0, 0, 0, 0.12); + box-shadow: inset 0 1px 3px rgba(0, 0, 0, 0.12); +} + +input[type=text]:focus, input[type=password]:focus { + border-color: #7dc9e2; + outline-color: #dceefc; + outline-offset: 0; +} + +input[type=button], input[type=submit] { + padding: 0 18px; + height: 29px; + font-size: 12px; + font-weight: bold; + color: #527881; + text-shadow: 0 1px #e3f1f1; + background: #cde5ef; + border: 1px solid; + border-color: #b4ccce #b3c0c8 #9eb9c2; + border-radius: 16px; + outline: 0; + -webkit-box-sizing: content-box; + -moz-box-sizing: content-box; + box-sizing: content-box; + background-image: -webkit-linear-gradient(top, #edf5f8, #cde5ef); + background-image: -moz-linear-gradient(top, #edf5f8, #cde5ef); + background-image: -o-linear-gradient(top, #edf5f8, #cde5ef); + background-image: linear-gradient(to bottom, #edf5f8, #cde5ef); + -webkit-box-shadow: inset 0 1px white, 0 1px 2px rgba(0, 0, 0, 0.15); + box-shadow: inset 0 1px white, 0 1px 2px rgba(0, 0, 0, 0.15); +} + +input[type=button]:active, input[type=submit]:active { + background: #cde5ef; + border-color: #9eb9c2 #b3c0c8 #b4ccce; + -webkit-box-shadow: inset 0 0 3px rgba(0, 0, 0, 0.2); + box-shadow: inset 0 0 3px rgba(0, 0, 0, 0.2); +} + +.lt-ie9 input[type=text], .lt-ie9 input[type=password] { + line-height: 34px; +} + +#room { + width: 100%; + text-align: center; +} + +#button-leave { + color: red; + text-align: center; + position: absolute; + bottom: 10px; +} + +.participant { + border-radius: 4px; + /* border: 2px groove; */ + margin-left: 5; + margin-right: 5; + width: 150; + text-align: center; + overflow: hide; + float: left; + padding: 5px; + border-radius: 10px; + -webkit-box-shadow: 0 0 200px rgba(255, 255, 255, 0.5), 0 1px 2px + rgba(0, 0, 0, 0.3); + box-shadow: 0 0 200px rgba(255, 255, 255, 0.5), 0 1px 2px + rgba(0, 0, 0, 0.3); + /*Transition*/ + -webkit-transition: all 0.3s linear; + -moz-transition: all 0.3s linear; + -o-transition: all 0.3s linear; + transition: all 0.3s linear; +} + +.participant:before { + content: ''; + position: absolute; + top: -8px; + right: -8px; + bottom: -8px; + left: -8px; + z-index: -1; + background: rgba(0, 0, 0, 0.08); + border-radius: 4px; +} + +.participant:hover { + opacity: 1; + background-color: 0A33B6; + -webkit-transition: all 0.5s linear; + transition: all 0.5s linear; +} + +.participant video, .participant.main video { + width: 100%; ! important; + height: auto; + ! + important; +} + +.participant span { + color: PapayaWhip; +} + +.participant.main { + width: 20%; + margin: 0 auto; +} + +.participant.main video { + height: auto; +} + +.animate { + -webkit-animation-duration: 0.5s; + -webkit-animation-fill-mode: both; + -moz-animation-duration: 0.5s; + -moz-animation-fill-mode: both; + -o-animation-duration: 0.5s; + -o-animation-fill-mode: both; + -ms-animation-duration: 0.5s; + -ms-animation-fill-mode: both; + animation-duration: 0.5s; + animation-fill-mode: both; +} + +.removed { + -webkit-animation: disapear 1s; + -webkit-animation-fill-mode: forwards; + animation: disapear 1s; + animation-fill-mode: forwards; +} + +@ +-webkit-keyframes disapear { 50% { + -webkit-transform: translateX(-5%); + transform: translateX(-5%); +} + +100% +{ +-webkit-transform + + + + + + +: + + + + + + + +translateX + + + + + + +(200%); +transform + + + + + + +: + + + + + + + +translateX + + + + + + +(200%); +} +} +@ +keyframes disapear { 50% { + -webkit-transform: translateX(-5%); + transform: translateX(-5%); +} + +100% +{ +-webkit-transform + + + + + + +: + + + + + + + +translateX + + + + + + +(200%); +transform + + + + + + +: + + + + + + + +translateX + + + + + + +(200%); +} +} +a.hovertext { + position: relative; + width: 500px; + text-decoration: none !important; + text-align: center; +} + +a.hovertext:after { + content: attr(title); + position: absolute; + left: 0; + bottom: 0; + padding: 0.5em 20px; + width: 460px; + background: rgba(0, 0, 0, 0.8); + text-decoration: none !important; + color: #fff; + opacity: 0; + -webkit-transition: 0.5s; + -moz-transition: 0.5s; + -o-transition: 0.5s; + -ms-transition: 0.5s; +} + +a.hovertext:hover:after, a.hovertext:focus:after { + opacity: 1.0; +} diff --git a/src/main/resources/static/group-call.html b/src/main/resources/static/group-call.html new file mode 100644 index 0000000..cc21201 --- /dev/null +++ b/src/main/resources/static/group-call.html @@ -0,0 +1,39 @@ + + + + + + + + + + + +
+
+
+

Join a Room

+
+

+ +

+

+ +

+

+ +

+
+
+ +
+
+ + diff --git a/src/main/resources/static/img/kurento.png b/src/main/resources/static/img/kurento.png new file mode 100644 index 0000000..6f1a4ad Binary files /dev/null and b/src/main/resources/static/img/kurento.png differ diff --git a/src/main/resources/static/img/naevatec.png b/src/main/resources/static/img/naevatec.png new file mode 100644 index 0000000..05ee704 Binary files /dev/null and b/src/main/resources/static/img/naevatec.png differ diff --git a/src/main/resources/static/img/pipeline.png b/src/main/resources/static/img/pipeline.png new file mode 100644 index 0000000..fad16bb Binary files /dev/null and b/src/main/resources/static/img/pipeline.png differ diff --git a/src/main/resources/static/img/spinner.gif b/src/main/resources/static/img/spinner.gif new file mode 100644 index 0000000..8be8ba3 Binary files /dev/null and b/src/main/resources/static/img/spinner.gif differ diff --git a/src/main/resources/static/img/transparent-1px.png b/src/main/resources/static/img/transparent-1px.png new file mode 100644 index 0000000..9da19ea Binary files /dev/null and b/src/main/resources/static/img/transparent-1px.png differ diff --git a/src/main/resources/static/img/urjc.gif b/src/main/resources/static/img/urjc.gif new file mode 100644 index 0000000..cd8a770 Binary files /dev/null and b/src/main/resources/static/img/urjc.gif differ diff --git a/src/main/resources/static/img/webrtc.png b/src/main/resources/static/img/webrtc.png new file mode 100644 index 0000000..d47e2e4 Binary files /dev/null and b/src/main/resources/static/img/webrtc.png differ diff --git a/src/main/resources/static/index.html b/src/main/resources/static/index.html index c767704..e7e7a28 100644 --- a/src/main/resources/static/index.html +++ b/src/main/resources/static/index.html @@ -1,52 +1,85 @@ - Hello WebSocket - - - - - + + + + + + + + + + + + + + + + + + + - -
+
+
+
+ +
+
+
+
-
-
-
- - - +
+ +
+
+
- -
-
-
-
- - +
+ Register +
+
+ +

+
+
+
- - +
+ Call Stop Play Rec + +
+
-
-
-
- - - - - - - - -
Greetings
+
+
+ +
+
+ +
+ - \ No newline at end of file + diff --git a/src/main/resources/static/js/app.js b/src/main/resources/static/js/app.js new file mode 100644 index 0000000..a143551 --- /dev/null +++ b/src/main/resources/static/js/app.js @@ -0,0 +1,303 @@ +const ws = new WebSocket('wss://' + location.host + '/helloworld'); +ws.onerror = (error) => { + console.error('WebSocket error:', error); + console.log('Detailed Error:', JSON.stringify(error, Object.getOwnPropertyNames(error))); +} + +let webRtcPeer; +// UI +let uiLocalVideo; +let uiRemoteVideo; +let uiState = null; +const UI_IDLE = 0; +const UI_STARTING = 1; +const UI_STARTED = 2; + +window.onload = function() +{ + uiLocalVideo = document.getElementById('uiLocalVideo'); + uiRemoteVideo = document.getElementById('uiRemoteVideo'); + uiStart() +} + +window.onbeforeunload = function() +{ + stop(); + console.log("Page unload - Close WebSocket"); + ws.close(); +} + +function explainUserMediaError(err) +{ + const n = err.name; + if (n === 'NotFoundError' || n === 'DevicesNotFoundError') { + return "Missing webcam for required tracks"; + } + else if (n === 'NotReadableError' || n === 'TrackStartError') { + return "Webcam is already in use"; + } + else if (n === 'OverconstrainedError' || n === 'ConstraintNotSatisfiedError') { + return "Webcam doesn't provide required tracks"; + } + else if (n === 'NotAllowedError' || n === 'PermissionDeniedError') { + return "Webcam permission has been denied by the user"; + } + else if (n === 'TypeError') { + return "No media tracks have been requested"; + } + else { + return "Unknown error: " + err; + } +} + +function sendError(message) +{ + console.error(message); + + sendMessage({ + id: 'ERROR', + message: message, + }); +} + +function sendMessage(message) +{ + if (ws.readyState !== ws.OPEN) { + console.warn("[sendMessage] Skip, WebSocket session isn't open"); + return; + } + + const jsonMessage = JSON.stringify(message); + console.log("[sendMessage] message: " + jsonMessage); + ws.send(jsonMessage); +} + +ws.onmessage = function(message) +{ + const jsonMessage = JSON.parse(message.data); + console.log("[onmessage] Received message: " + message.data); + + switch (jsonMessage.id) { + case 'PROCESS_SDP_ANSWER': + handleProcessSdpAnswer(jsonMessage); + break; + case 'ADD_ICE_CANDIDATE': + handleAddIceCandidate(jsonMessage); + break; + case 'ERROR': + handleError(jsonMessage); + break; + default: + // Ignore the message + console.warn("[onmessage] Invalid message, id: " + jsonMessage.id); + break; + } +} + +// PROCESS_SDP_ANSWER ---------------------------------------------------------- + +function handleProcessSdpAnswer(jsonMessage) +{ + console.log("[handleProcessSdpAnswer] SDP Answer from Kurento, process in WebRTC Peer"); + + if (webRtcPeer == null) { + console.warn("[handleProcessSdpAnswer] Skip, no WebRTC Peer"); + return; + } + + webRtcPeer.processAnswer(jsonMessage.sdpAnswer, (err) => { + if (err) { + sendError("[handleProcessSdpAnswer] Error: " + err); + stop(); + return; + } + + console.log("[handleProcessSdpAnswer] SDP Answer ready; start remote video"); + startVideo(uiRemoteVideo); + + uiSetState(UI_STARTED); + }); +} + +// ADD_ICE_CANDIDATE ----------------------------------------------------------- + +function handleAddIceCandidate(jsonMessage) +{ + if (webRtcPeer == null) { + console.warn("[handleAddIceCandidate] Skip, no WebRTC Peer"); + return; + } + + webRtcPeer.addIceCandidate(jsonMessage.candidate, (err) => { + if (err) { + console.error("[handleAddIceCandidate] " + err); + return; + } + }); +} + +// STOP ------------------------------------------------------------------------ + +function stop() +{ + if (uiState == UI_IDLE) { + console.log("[stop] Skip, already stopped"); + return; + } + + console.log("[stop]"); + + if (webRtcPeer) { + webRtcPeer.dispose(); + webRtcPeer = null; + } + + uiSetState(UI_IDLE); + hideSpinner(uiLocalVideo, uiRemoteVideo); + + sendMessage({ + id: 'STOP', + }); +} + +// ERROR ----------------------------------------------------------------------- + +function handleError(jsonMessage) +{ + const errMessage = jsonMessage.message; + console.error("Kurento error: " + errMessage); + + console.log("Assume that the other side stops after an error..."); + stop(); +} + +function uiStart() +{ + console.log("[start] Create WebRtcPeerSendrecv"); + uiSetState(UI_STARTING); + showSpinner(uiLocalVideo, uiRemoteVideo); + + const options = { + localVideo: uiLocalVideo, + remoteVideo: uiRemoteVideo, + mediaConstraints: { audio: true, video: true }, + onicecandidate: (candidate) => sendMessage({ + id: 'ADD_ICE_CANDIDATE', + candidate: candidate, + }), + }; + + options.configuration = { + // iceServers : JSON.parse('[{"urls":"stun:stun.l.google.com:19302"}]') + iceServers: JSON.parse('[{"urls":"stun:stun.l.google.com:19302"}, {"urls":"turns:standard.relay.metered.ca:80", ' + + '"username":"65024e9d0265012cc6669435", ' + + '"credential":"k3nUiqAyH5SM/5qt"}]') + }; + webRtcPeer = new kurentoUtils.WebRtcPeer.WebRtcPeerSendrecv(options, + function(err) + { + if (err) { + sendError("[start/WebRtcPeerSendrecv] Error: " + + explainUserMediaError(err)); + stop(); + return; + } + + console.log("[start/WebRtcPeerSendrecv] Created; start local video"); + startVideo(uiLocalVideo); + + console.log("[start/WebRtcPeerSendrecv] Generate SDP Offer"); + webRtcPeer.generateOffer((err, sdpOffer) => { + if (err) { + sendError("[start/WebRtcPeerSendrecv/generateOffer] Error: " + err); + stop(); + return; + } + + sendMessage({ + id: 'PROCESS_SDP_OFFER', + sdpOffer: sdpOffer, + }); + + console.log("[start/WebRtcPeerSendrecv/generateOffer] Done!"); + uiSetState(UI_STARTED); + }); + }); +} + +// Stop ------------------------------------------------------------------------ + +function uiStop() +{ + stop(); +} + +function uiSetState(newState) +{ + switch (newState) { + case UI_IDLE: + uiEnableElement('#uiStartBtn', 'uiStart()'); + uiDisableElement('#uiStopBtn'); + break; + case UI_STARTING: + uiDisableElement('#uiStartBtn'); + uiDisableElement('#uiStopBtn'); + break; + case UI_STARTED: + uiDisableElement('#uiStartBtn'); + uiEnableElement('#uiStopBtn', 'uiStop()'); + break; + default: + console.warn("[setState] Skip, invalid state: " + newState); + return; + } + uiState = newState; +} + +function uiEnableElement(id, onclickHandler) +{ + $(id).attr('disabled', false); + if (onclickHandler) { + $(id).attr('onclick', onclickHandler); + } +} + +function uiDisableElement(id) +{ + $(id).attr('disabled', true); + $(id).removeAttr('onclick'); +} + +function showSpinner() +{ + for (let i = 0; i < arguments.length; i++) { + arguments[i].poster = './img/transparent-1px.png'; + arguments[i].style.background = "center transparent url('./img/spinner.gif') no-repeat"; + } +} + +function hideSpinner() +{ + for (let i = 0; i < arguments.length; i++) { + arguments[i].src = ''; + arguments[i].poster = './img/webrtc.png'; + arguments[i].style.background = ''; + } +} + +function startVideo(video) +{ + video.play().catch((err) => { + if (err.name === 'NotAllowedError') { + console.error("[start] Browser doesn't allow playing video: " + err); + } + else { + console.error("[start] Error in video.play(): " + err); + } + }); +} + +$(document).delegate('*[data-toggle="lightbox"]', 'click', function(event) { + event.preventDefault(); + $(this).ekkoLightbox(); +}); diff --git a/src/main/resources/static/js/call.js b/src/main/resources/static/js/call.js new file mode 100644 index 0000000..0bce6ec --- /dev/null +++ b/src/main/resources/static/js/call.js @@ -0,0 +1,22 @@ +// Basic logic for controls +const muteButton = document.getElementById("mute"); +const cameraButton = document.getElementById("camera"); +const endCallButton = document.getElementById("endCall"); + +let isMuted = false; +let isCameraOn = true; + +muteButton.addEventListener("click", () => { + isMuted = !isMuted; + muteButton.textContent = isMuted ? "🔇" : "🎤"; +}); + +cameraButton.addEventListener("click", () => { + isCameraOn = !isCameraOn; + cameraButton.textContent = isCameraOn ? "📷" : "📴"; +}); + +endCallButton.addEventListener("click", () => { + alert("Call ended"); + // Add logic to terminate the call +}); diff --git a/src/main/resources/static/js/conferenceroom.js b/src/main/resources/static/js/conferenceroom.js new file mode 100644 index 0000000..8a93d08 --- /dev/null +++ b/src/main/resources/static/js/conferenceroom.js @@ -0,0 +1,166 @@ +var ws = new WebSocket('wss://' + location.host + '/groupcall'); +var participants = {}; +var name; + +window.onbeforeunload = function() { + ws.close(); +}; + +ws.onmessage = function(message) { + var parsedMessage = JSON.parse(message.data); + console.info('Received message: ' + message.data); + + switch (parsedMessage.id) { + case 'existingParticipants': + onExistingParticipants(parsedMessage); + break; + case 'newParticipantArrived': + onNewParticipant(parsedMessage); + break; + case 'participantLeft': + onParticipantLeft(parsedMessage); + break; + case 'receiveVideoAnswer': + receiveVideoResponse(parsedMessage); + break; + case 'iceCandidate': + participants[parsedMessage.name].rtcPeer.addIceCandidate(parsedMessage.candidate, function (error) { + if (error) { + console.error("Error adding candidate: " + error); + return; + } + }); + break; + default: + console.error('Unrecognized message', parsedMessage); + } +} + +function register() { + name = document.getElementById('name').value; + var room = document.getElementById('roomName').value; + + document.getElementById('room-header').innerText = 'ROOM ' + room; + document.getElementById('join').style.display = 'none'; + document.getElementById('room').style.display = 'block'; + + var message = { + id : 'joinRoom', + name : name, + room : room, + } + sendMessage(message); +} + +function onNewParticipant(request) { + receiveVideo(request.name); +} + +function receiveVideoResponse(result) { + participants[result.name].rtcPeer.processAnswer (result.sdpAnswer, function (error) { + if (error) return console.error (error); + }); +} + +function callResponse(message) { + if (message.response != 'accepted') { + console.info('Call not accepted by peer. Closing call'); + stop(); + } else { + webRtcPeer.processAnswer(message.sdpAnswer, function (error) { + if (error) return console.error (error); + }); + } +} + +function onExistingParticipants(msg) { + var constraints = { + audio : true, + video : { + mandatory : { + maxWidth : 320, + maxFrameRate : 15, + minFrameRate : 15 + } + } + }; + console.log(name + " registered in room " + room); + var participant = new Participant(name); + participants[name] = participant; + var video = participant.getVideoElement(); + + var options = { + localVideo: video, + mediaConstraints: constraints, + onicecandidate: participant.onIceCandidate.bind(participant) + } + options.configuration = { + // iceServers : JSON.parse('[{"urls":"stun:stun.l.google.com:19302"}]') + iceServers: JSON.parse('[{"urls":"stun:stun.l.google.com:19302"}, {"urls":"turns:standard.relay.metered.ca:80", ' + + '"username":"65024e9d0265012cc6669435", ' + + '"credential":"k3nUiqAyH5SM/5qt"}]') + }; + participant.rtcPeer = new kurentoUtils.WebRtcPeer.WebRtcPeerSendonly(options, + function (error) { + if(error) { + return console.error(error); + } + this.generateOffer (participant.offerToReceiveVideo.bind(participant)); + }); + + msg.data.forEach(receiveVideo); +} + +function leaveRoom() { + sendMessage({ + id : 'leaveRoom' + }); + + for ( var key in participants) { + participants[key].dispose(); + } + + document.getElementById('join').style.display = 'block'; + document.getElementById('room').style.display = 'none'; + + ws.close(); +} + +function receiveVideo(sender) { + var participant = new Participant(sender); + participants[sender] = participant; + var video = participant.getVideoElement(); + + var options = { + remoteVideo: video, + onicecandidate: participant.onIceCandidate.bind(participant) + } + + options.configuration = { + // iceServers : JSON.parse('[{"urls":"stun:stun.l.google.com:19302"}]') + iceServers: JSON.parse('[{"urls":"stun:stun.l.google.com:19302"}, {"urls":"turns:standard.relay.metered.ca:80", ' + + '"username":"65024e9d0265012cc6669435", ' + + '"credential":"k3nUiqAyH5SM/5qt"}]') + }; + + participant.rtcPeer = new kurentoUtils.WebRtcPeer.WebRtcPeerRecvonly(options, + function (error) { + if(error) { + return console.error(error); + } + this.generateOffer (participant.offerToReceiveVideo.bind(participant)); + });; +} + +function onParticipantLeft(request) { + console.log('Participant ' + request.name + ' left'); + var participant = participants[request.name]; + participant.dispose(); + delete participants[request.name]; +} + +function sendMessage(message) { + var jsonMessage = JSON.stringify(message); + console.log('Sending message: ' + jsonMessage); + ws.send(jsonMessage); +} diff --git a/src/main/resources/static/js/participant.js b/src/main/resources/static/js/participant.js new file mode 100644 index 0000000..dce94b3 --- /dev/null +++ b/src/main/resources/static/js/participant.js @@ -0,0 +1,79 @@ +const PARTICIPANT_MAIN_CLASS = 'participant main'; +const PARTICIPANT_CLASS = 'participant'; + +function Participant(name) { + this.name = name; + var container = document.createElement('div'); + container.className = isPresentMainParticipant() ? PARTICIPANT_CLASS : PARTICIPANT_MAIN_CLASS; + container.id = name; + var span = document.createElement('span'); + var video = document.createElement('video'); + var rtcPeer; + + container.appendChild(video); + container.appendChild(span); + container.onclick = switchContainerClass; + document.getElementById('participants').appendChild(container); + + span.appendChild(document.createTextNode(name)); + + video.id = 'video-' + name; + video.autoplay = true; + video.controls = false; + + + this.getElement = function() { + return container; + } + + this.getVideoElement = function() { + return video; + } + + function switchContainerClass() { + if (container.className === PARTICIPANT_CLASS) { + var elements = Array.prototype.slice.call(document.getElementsByClassName(PARTICIPANT_MAIN_CLASS)); + elements.forEach(function(item) { + item.className = PARTICIPANT_CLASS; + }); + + container.className = PARTICIPANT_MAIN_CLASS; + } else { + container.className = PARTICIPANT_CLASS; + } + } + + function isPresentMainParticipant() { + return ((document.getElementsByClassName(PARTICIPANT_MAIN_CLASS)).length != 0); + } + + this.offerToReceiveVideo = function(error, offerSdp, wp){ + if (error) return console.error ("sdp offer error") + console.log('Invoking SDP offer callback function'); + var msg = { id : "receiveVideoFrom", + sender : name, + sdpOffer : offerSdp + }; + sendMessage(msg); + } + + + this.onIceCandidate = function (candidate, wp) { + console.log("Local candidate" + JSON.stringify(candidate)); + + var message = { + id: 'onIceCandidate', + candidate: candidate, + name: name + }; + sendMessage(message); + } + + Object.defineProperty(this, 'rtcPeer', { writable: true}); + + this.dispose = function() { + console.log('Disposing participant ' + this.name); + this.rtcPeer.dispose(); + container.parentNode.removeChild(container); + }; +} diff --git a/src/main/resources/static/js/recording.js b/src/main/resources/static/js/recording.js new file mode 100644 index 0000000..1aa0e45 --- /dev/null +++ b/src/main/resources/static/js/recording.js @@ -0,0 +1,403 @@ +/* + * (C) Copyright 2014 Kurento (http://kurento.org/) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +var ws = new WebSocket('wss://' + location.host + '/call'); +var videoInput; +var videoOutput; +var webRtcPeer; +var from; + +var registerName = null; +var registerState = null; +const NOT_REGISTERED = 0; +const REGISTERING = 1; +const REGISTERED = 2; + +function setRegisterState(nextState) { + switch (nextState) { + case NOT_REGISTERED: + enableButton('#register', 'register()'); + setCallState(DISABLED); + break; + case REGISTERING: + disableButton('#register'); + break; + case REGISTERED: + disableButton('#register'); + setCallState(NO_CALL); + break; + default: + return; + } + registerState = nextState; +} + +var callState = null; +const NO_CALL = 0; +const IN_CALL = 1; +const POST_CALL = 2; +const DISABLED = 3; +const IN_PLAY = 4; + +function setCallState(nextState) { + switch (nextState) { + case NO_CALL: + enableButton('#call', 'call()'); + disableButton('#terminate'); + disableButton('#play'); + break; + case DISABLED: + disableButton('#call'); + disableButton('#terminate'); + disableButton('#play'); + break; + case POST_CALL: + enableButton('#call', 'call()'); + disableButton('#terminate'); + enableButton('#play', 'play()'); + break; + case IN_CALL: + case IN_PLAY: + disableButton('#call'); + enableButton('#terminate', 'stop()'); + disableButton('#play'); + break; + default: + return; + } + callState = nextState; +} + +function disableButton(id) { + $(id).attr('disabled', true); + $(id).removeAttr('onclick'); +} + +function enableButton(id, functionName) { + $(id).attr('disabled', false); + $(id).attr('onclick', functionName); +} + +window.onload = function() { + setRegisterState(NOT_REGISTERED); + var drag = new Draggabilly(document.getElementById('videoSmall')); + videoInput = document.getElementById('videoInput'); + videoOutput = document.getElementById('videoOutput'); + document.getElementById('name').focus(); +} + +window.onbeforeunload = function() { + ws.close(); +} + +ws.onmessage = function(message) { + var parsedMessage = JSON.parse(message.data); + console.info('Received message: ' + message.data); + + switch (parsedMessage.id) { + case 'registerResponse': + registerResponse(parsedMessage); + break; + case 'callResponse': + callResponse(parsedMessage); + break; + case 'incomingCall': + incomingCall(parsedMessage); + break; + case 'startCommunication': + startCommunication(parsedMessage); + break; + case 'stopCommunication': + console.info('Communication ended by remote peer'); + stop(true); + break; + case 'playResponse': + playResponse(parsedMessage); + break; + case 'playEnd': + playEnd(); + break; + case 'iceCandidate': + webRtcPeer.addIceCandidate(parsedMessage.candidate, function(error) { + if (error) + return console.error('Error adding candidate: ' + error); + }); + break; + default: + console.error('Unrecognized message', parsedMessage); + } +} + +function registerResponse(message) { + if (message.response == 'accepted') { + setRegisterState(REGISTERED); + document.getElementById('peer').focus(); + } else { + setRegisterState(NOT_REGISTERED); + var errorMessage = message.response ? message.response + : 'Unknown reason for register rejection.'; + console.log(errorMessage); + document.getElementById('name').focus(); + alert('Error registering user. See console for further information.'); + } +} + +function callResponse(message) { + if (message.response != 'accepted') { + console.info('Call not accepted by peer. Closing call'); + stop(); + setCallState(NO_CALL); + if (message.message) { + alert(message.message); + } + } else { + setCallState(IN_CALL); + webRtcPeer.processAnswer(message.sdpAnswer, function(error) { + if (error) + return console.error(error); + }); + } +} + +function startCommunication(message) { + setCallState(IN_CALL); + webRtcPeer.processAnswer(message.sdpAnswer, function(error) { + if (error) + return console.error(error); + }); +} + +function playResponse(message) { + if (message.response != 'accepted') { + hideSpinner(videoOutput); + document.getElementById('videoSmall').style.display = 'block'; + alert(message.error); + document.getElementById('peer').focus(); + setCallState(POST_CALL); + } else { + setCallState(IN_PLAY); + webRtcPeer.processAnswer(message.sdpAnswer, function(error) { + if (error) + return console.error(error); + }); + } +} + +function incomingCall(message) { + // If bussy just reject without disturbing user + if (callState != NO_CALL && callState != POST_CALL) { + var response = { + id : 'incomingCallResponse', + from : message.from, + callResponse : 'reject', + message : 'bussy' + }; + return sendMessage(response); + } + + setCallState(DISABLED); + if (confirm('User ' + message.from + + ' is calling you. Do you accept the call?')) { + showSpinner(videoInput, videoOutput); + + from = message.from; + var options = { + localVideo : videoInput, + remoteVideo : videoOutput, + onicecandidate : onIceCandidate + } + webRtcPeer = new kurentoUtils.WebRtcPeer.WebRtcPeerSendrecv(options, + function(error) { + if (error) { + return console.error(error); + } + this.generateOffer(onOfferIncomingCall); + }); + } else { + var response = { + id : 'incomingCallResponse', + from : message.from, + callResponse : 'reject', + message : 'user declined' + }; + sendMessage(response); + stop(); + } +} + +function onOfferIncomingCall(error, offerSdp) { + if (error) + return console.error('Error generating the offer ' + error); + var response = { + id : 'incomingCallResponse', + from : from, + callResponse : 'accept', + sdpOffer : offerSdp + }; + sendMessage(response); +} + +function register() { + var name = document.getElementById('name').value; + if (name == '') { + window.alert('You must insert your user name'); + document.getElementById('name').focus(); + return; + } + setRegisterState(REGISTERING); + + var message = { + id : 'register', + name : name + }; + sendMessage(message); +} + +function call() { + if (document.getElementById('peer').value == '') { + document.getElementById('peer').focus(); + window.alert('You must specify the peer name'); + return; + } + setCallState(DISABLED); + showSpinner(videoInput, videoOutput); + + var options = { + localVideo : videoInput, + remoteVideo : videoOutput, + onicecandidate : onIceCandidate + } + webRtcPeer = new kurentoUtils.WebRtcPeer.WebRtcPeerSendrecv(options, + function(error) { + if (error) { + return console.error(error); + } + this.generateOffer(onOfferCall); + }); +} + +function onOfferCall(error, offerSdp) { + if (error) + return console.error('Error generating the offer ' + error); + console.log('Invoking SDP offer callback function'); + var message = { + id : 'call', + from : document.getElementById('name').value, + to : document.getElementById('peer').value, + sdpOffer : offerSdp + }; + sendMessage(message); +} + +function play() { + var peer = document.getElementById('peer').value; + if (peer == '') { + window + .alert("You must insert the name of the user recording to be played (field 'Peer')"); + document.getElementById('peer').focus(); + return; + } + + document.getElementById('videoSmall').style.display = 'none'; + setCallState(DISABLED); + showSpinner(videoOutput); + + var options = { + remoteVideo : videoOutput, + onicecandidate : onIceCandidate + } + webRtcPeer = new kurentoUtils.WebRtcPeer.WebRtcPeerRecvonly(options, + function(error) { + if (error) { + return console.error(error); + } + this.generateOffer(onOfferPlay); + }); +} + +function onOfferPlay(error, offerSdp) { + console.log('Invoking SDP offer callback function'); + var message = { + id : 'play', + user : document.getElementById('peer').value, + sdpOffer : offerSdp + }; + sendMessage(message); +} + +function playEnd() { + setCallState(POST_CALL); + hideSpinner(videoInput, videoOutput); + document.getElementById('videoSmall').style.display = 'block'; +} + +function stop(message) { + var stopMessageId = (callState == IN_CALL) ? 'stop' : 'stopPlay'; + setCallState(POST_CALL); + if (webRtcPeer) { + webRtcPeer.dispose(); + webRtcPeer = null; + + if (!message) { + var message = { + id : stopMessageId + } + sendMessage(message); + } + } + hideSpinner(videoInput, videoOutput); + document.getElementById('videoSmall').style.display = 'block'; +} + +function sendMessage(message) { + var jsonMessage = JSON.stringify(message); + console.log('Sending message: ' + jsonMessage); + ws.send(jsonMessage); +} + +function onIceCandidate(candidate) { + console.log('Local candidate ' + JSON.stringify(candidate)); + + var message = { + id : 'onIceCandidate', + candidate : candidate + }; + sendMessage(message); +} + +function showSpinner() { + for (var i = 0; i < arguments.length; i++) { + arguments[i].poster = './img/transparent-1px.png'; + arguments[i].style.background = 'center transparent url("./img/spinner.gif") no-repeat'; + } +} + +function hideSpinner() { + for (var i = 0; i < arguments.length; i++) { + arguments[i].src = ''; + arguments[i].poster = './img/webrtc.png'; + arguments[i].style.background = ''; + } +} + +/** + * Lightbox utility (to display media pipeline image in a modal dialog) + */ +$(document).delegate('*[data-toggle="lightbox"]', 'click', function(event) { + event.preventDefault(); + $(this).ekkoLightbox(); +}); diff --git a/src/main/resources/static/one-to-one.html b/src/main/resources/static/one-to-one.html new file mode 100644 index 0000000..2537d2a --- /dev/null +++ b/src/main/resources/static/one-to-one.html @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+ +
+ + + +
+
+ + + + diff --git a/src/main/resources/static/video/trump.mp4 b/src/main/resources/static/video/trump.mp4 new file mode 100644 index 0000000..93ac858 Binary files /dev/null and b/src/main/resources/static/video/trump.mp4 differ diff --git a/src/test/java/io/openfuture/openmessanger/OpenMessangerApplicationTests.java b/src/test/java/io/openfuture/openmessanger/OpenMessangerApplicationTests.java deleted file mode 100644 index 3b39ae3..0000000 --- a/src/test/java/io/openfuture/openmessanger/OpenMessangerApplicationTests.java +++ /dev/null @@ -1,13 +0,0 @@ -package io.openfuture.openmessanger; - -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest -class OpenMessangerApplicationTests { - - @Test - void contextLoads() { - } - -} diff --git a/src/test/java/io/openfuture/openmessenger/OpenMessengerApplicationTests.kt b/src/test/java/io/openfuture/openmessenger/OpenMessengerApplicationTests.kt new file mode 100644 index 0000000..dba5261 --- /dev/null +++ b/src/test/java/io/openfuture/openmessenger/OpenMessengerApplicationTests.kt @@ -0,0 +1,22 @@ +package io.openfuture.openmessenger + +import org.junit.jupiter.api.Test +import org.kurento.client.KurentoClient +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.mock.mockito.MockBean +import org.springframework.test.context.ContextConfiguration +import org.springframework.test.context.web.WebAppConfiguration + +@WebAppConfiguration +@SpringBootTest +@ContextConfiguration(initializers = [TestInitializer::class]) +class OpenMessengerApplicationTests { + + @MockBean + lateinit var kurentoClient: KurentoClient + + @Test + fun contextLoads() { + } + +} \ No newline at end of file diff --git a/src/test/java/io/openfuture/openmessenger/TestInitializer.kt b/src/test/java/io/openfuture/openmessenger/TestInitializer.kt new file mode 100644 index 0000000..f203930 --- /dev/null +++ b/src/test/java/io/openfuture/openmessenger/TestInitializer.kt @@ -0,0 +1,38 @@ +package io.openfuture.openmessenger + +import org.springframework.context.ApplicationContextInitializer +import org.springframework.context.ConfigurableApplicationContext +import org.springframework.context.annotation.Configuration +import org.springframework.test.context.support.TestPropertySourceUtils +import org.testcontainers.containers.PostgreSQLContainer +import org.testcontainers.utility.DockerImageName + +@Configuration +class TestInitializer : ApplicationContextInitializer { + + private fun lazyInit() { + synchronized(CONTAINER) { + if (!CONTAINER.isRunning) { + CONTAINER.start() + } + } + } + + companion object { + private val CONTAINER: PostgreSQLContainer<*> = PostgreSQLContainer(DockerImageName.parse("postgres:latest")) + .withDatabaseName("open_chat") + .withUsername("postgres") + .withPassword("123456") + } + + override fun initialize(applicationContext: ConfigurableApplicationContext) { + lazyInit() + TestPropertySourceUtils.addInlinedPropertiesToEnvironment( + applicationContext!!, + "spring.datasource.url=" + CONTAINER.jdbcUrl, + "spring.datasource.username=" + CONTAINER.username, + "spring.datasource.password=" + CONTAINER.password + ) + } +} +