diff --git a/.devcontainer.json b/.devcontainer.json new file mode 100644 index 000000000..4542886ed --- /dev/null +++ b/.devcontainer.json @@ -0,0 +1,3 @@ +{ + "image": "maven:3-eclipse-temurin-11" +} \ No newline at end of file diff --git a/.github/workflows/Manual Release.yml b/.github/workflows/Manual Release.yml new file mode 100644 index 000000000..0152c8c32 --- /dev/null +++ b/.github/workflows/Manual Release.yml @@ -0,0 +1,74 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Publish Release + +on: + schedule: + # At 0:00am each day on the default branch + - cron: '0 0 * * *' + workflow_dispatch: + inputs: + tag: + description: 'Version tag' + required: true + type: string + name: + description: 'Release name' + required: true + type: string + skip_tests: + description: 'Skip tests' + required: true + default: false + type: boolean + +jobs: + publish_release: + runs-on: ubuntu-latest + container: maven:3-eclipse-temurin-8 + steps: + - uses: actions/checkout@v3 + + - name: Cache local Maven repository + uses: actions/cache/restore@v3 + with: + path: /root/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build and Test with Maven + run: mvn -P '!add-dependencies-for-IDEA,full-build,include-front-end' -B package -DskipTests=${{ github.event_name == 'workflow_dispatch' && inputs.skip_tests || true }} --file flink-cyber/pom.xml + + - name: Release + uses: softprops/action-gh-release@v2 + if: ${{ github.event_name == 'workflow_dispatch' }} + with: + tag_name: ${{ inputs.tag }} + name: ${{ inputs.name }} + draft: true + generate_release_notes: true + token: ${{ secrets.GITHUB_TOKEN }} + files: | + flink-cyber/cyber-csd/target/cyber-csd--* + flink-cyber/cyber-csd/target/CYBERSEC-* + flink-cyber/cyber-parcel/target/CYBERSEC-* + + - name: Get current date + id: date + run: echo "CURRENT_DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV + + - name: Publish artifact + uses: actions/upload-artifact@v4 + with: + name: cybersec-${{ github.event_name == 'workflow_dispatch' && 'release' || 'nightly' }}-${{ env.CURRENT_DATE }} + path: | + flink-cyber/cyber-csd/target/cyber-csd--* + flink-cyber/cyber-csd/target/CYBERSEC-* + flink-cyber/cyber-parcel/target/CYBERSEC-* diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml new file mode 100644 index 000000000..c2dd13f88 --- /dev/null +++ b/.github/workflows/build_and_test.yml @@ -0,0 +1,53 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Build and Test + +on: + push: + branches: [ "develop" ] + pull_request: + branches: [ "develop" ] + +jobs: + build_and_test_java: + runs-on: ubuntu-latest + container: maven:3-eclipse-temurin-8 + steps: + - uses: actions/checkout@v3 + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: /root/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Build and Test with Maven + run: mvn -P '!add-dependencies-for-IDEA,!full-build,!include-front-end' -B package --file flink-cyber/pom.xml + + # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive + - name: Update dependency graph + uses: advanced-security/maven-dependency-submission-action@v3 + with: + directory: /home/runner/work/cybersec/cybersec/flink-cyber + build_and_test_angular: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version: '20.x' + - name: Install Angular CLI + run: npm install -g @angular/cli + - name: Install dependencies + run: npm install + working-directory: ./flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client + - name: Run test + run: npm run test:headless + working-directory: ./flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client diff --git a/.github/workflows/build_and_test_future_java.yml b/.github/workflows/build_and_test_future_java.yml new file mode 100644 index 000000000..3e1150f90 --- /dev/null +++ b/.github/workflows/build_and_test_future_java.yml @@ -0,0 +1,60 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Future Java Compatibility Test + +on: + schedule: +# At 2:30am each Saturday on the default branch + - cron: '30 2 * * 6' + +jobs: + build_and_test_java11: + runs-on: ubuntu-latest + container: maven:3-eclipse-temurin-11 + steps: + - uses: actions/checkout@v3 + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Build and Test with Maven + run: mvn -B package --file flink-cyber/pom.xml + + build_and_test_java17: + runs-on: ubuntu-latest + container: maven:3-eclipse-temurin-17 + steps: + - uses: actions/checkout@v3 + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Build and Test with Maven + run: mvn -B package --file flink-cyber/pom.xml + + build_and_test_java21: + runs-on: ubuntu-latest + container: maven:3-eclipse-temurin-21 + steps: + - uses: actions/checkout@v3 + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Build and Test with Maven + run: mvn -B package --file flink-cyber/pom.xml diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml new file mode 100644 index 000000000..a131ef3b5 --- /dev/null +++ b/.github/workflows/publish_release.yml @@ -0,0 +1,92 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Publish Release + +on: + schedule: + # At 0:00am each day on the default branch + - cron: '0 0 * * *' + workflow_dispatch: + inputs: + tag: + description: 'Version tag' + required: true + type: string + name: + description: 'Release name' + required: true + type: string + skip_tests: + description: 'Skip tests' + required: true + default: false + type: boolean + +jobs: + # checks if there's a commit in last 24hr + check_date: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: print latest_commit + run: echo ${{ github.sha }} + + - id: should_run + continue-on-error: true + name: check latest commit is less than a day + run: | + test -z $(git rev-list --after="24 hours" ${{ github.sha }}) \ + && (gh run cancel ${{ github.run_id }} \ + && gh run watch ${{ github.run_id }}) + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + publish_release: + needs: check_date + runs-on: ubuntu-latest + container: maven:3-eclipse-temurin-8 + steps: + - uses: actions/checkout@v3 + + - name: Cache local Maven repository + uses: actions/cache/restore@v3 + with: + path: /root/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Build and Test with Maven + run: mvn -P '!add-dependencies-for-IDEA,full-build,include-front-end' -B package -DskipTests=${{ github.event_name == 'workflow_dispatch' && inputs.skip_tests || true }} --file flink-cyber/pom.xml + + - name: Release + uses: softprops/action-gh-release@v2 + if: ${{ github.event_name == 'workflow_dispatch' }} + with: + tag_name: ${{ inputs.tag }} + name: ${{ inputs.name }} + draft: true + generate_release_notes: true + token: ${{ secrets.GITHUB_TOKEN }} + files: | + flink-cyber/cyber-csd/target/cyber-csd--* + flink-cyber/cyber-csd/target/CYBERSEC-* + flink-cyber/cyber-parcel/target/CYBERSEC-* + + - name: Get current date + id: date + run: echo "CURRENT_DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV + + - name: Publish artifact + uses: actions/upload-artifact@v4 + with: + name: cybersec-${{ github.event_name == 'workflow_dispatch' && 'release' || 'nightly' }}-${{ env.CURRENT_DATE }} + path: | + flink-cyber/cyber-csd/target/cyber-csd--* + flink-cyber/cyber-csd/target/CYBERSEC-* + flink-cyber/cyber-parcel/target/CYBERSEC-* diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..4128fe598 --- /dev/null +++ b/.gitignore @@ -0,0 +1,150 @@ +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +# https://github.com/takari/maven-wrapper#usage-without-binary-jar +.mvn/wrapper/maven-wrapper.jar + +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/workspace.xml +.idea/sonarlint +.idea/cybersec.iml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf +.idea/encodings.xml +.idea/misc.xml +.idea/inspectionProfiles/Project_Default.xml +.idea/modules.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries +.idea/workspace.xml + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +auto-import. +.idea +**/.angular +*.iml +*.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + + +.metadata +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath +.recommenders + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +.project +/flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/generated/* \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..16a17022c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "flink-solr-log-indexer"] + path = flink-solr-log-indexer + url = https://github.com/simonellistonball/flink-solr-log-indexer.git diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml deleted file mode 100644 index a55e7a179..000000000 --- a/.idea/codeStyles/codeStyleConfig.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/cybersec.iml b/.idea/cybersec.iml deleted file mode 100644 index d6ebd4805..000000000 --- a/.idea/cybersec.iml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 2816ac0a0..000000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddfb..000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index ba57f15ff..000000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - -