feat: Initial release

maheckathorn · maheckathorn · commit 9852c069ccb2 · 2024-11-01T14:15:57.000-04:00
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,7 @@
+---
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,116 @@
+---
+name: Release
+'on':
+  pull_request:
+  push:
+    branches:
+      - main
+  schedule:
+      - cron: "0 7 * * 0"
+
+env:
+  IMAGE_NAME: fixbuf
+
+jobs:
+
+  # Test the image builds and works correctly.
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out the codebase.
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Install test dependencies.
+        run: pip3 install pytest-testinfra
+
+      - name: Build image.
+        run: docker build -t cmusei/${{ env.IMAGE_NAME }} .
+
+      - name: Run the built image.
+        run: docker run --name=${{ env.IMAGE_NAME }} -td cmusei/${{ env.IMAGE_NAME }}
+
+      - name: Test the built image.
+        run: py.test --hosts='docker://${{ env.IMAGE_NAME }}'
+
+  # If on main branch, build and release image.
+  release2:
+    name: Release2
+    runs-on: ubuntu-latest
+    needs: test
+    if: github.ref == 'refs/heads/main'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}          
+
+      - name: Build and push image.
+        uses: docker/build-push-action@v6
+        with:
+          context: ./
+          file: Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            cmusei/${{ env.IMAGE_NAME }}:latest
+            cmusei/${{ env.IMAGE_NAME }}:2
+            cmusei/${{ env.IMAGE_NAME }}:2.5.0
+            ghcr.io/cmu-sei/${{ env.IMAGE_NAME }}:latest
+            ghcr.io/cmu-sei/${{ env.IMAGE_NAME }}:2
+            ghcr.io/cmu-sei/${{ env.IMAGE_NAME }}:2.5.0
+  release3:
+    name: Release3
+    runs-on: ubuntu-latest
+    needs: test
+    if: github.ref == 'refs/heads/main'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}          
+
+      - name: Build and push image.
+        uses: docker/build-push-action@v6
+        with:
+          context: ./
+          file: Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          build-args: |
+            FIXBUF_VERSION=3.0.0.alpha2
+          tags: |
+            cmusei/${{ env.IMAGE_NAME }}:3
+            ghcr.io/cmu-sei/${{ env.IMAGE_NAME }}:3
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+__pycache__/
+inventory
+pytest_junit.xml
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,46 @@
+FROM debian:11 AS build
+
+# Specify software versions to download
+ARG FIXBUF_VERSION=2.5.0
+
+# Pre-reqs:
+# curl for downloading
+# build-essentials for build tools
+# ca-certs to download https
+# glib2.0 for fixbuf
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl \
+        build-essential \
+        ca-certificates \
+        libssl-dev \
+        libglib2.0-dev \
+        && apt-get clean && \
+        rm -rf /var/lib/apt/lists/*
+
+RUN mkdir /netsa
+WORKDIR /netsa
+
+# Download & Install fixbuf
+RUN curl https://tools.netsa.cert.org/releases/libfixbuf-$FIXBUF_VERSION.tar.gz | \
+        tar -xz && cd libfixbuf-* && \
+        ./configure --prefix=/netsa \
+        --with-openssl && \
+        make && \
+        make install && \
+        cd ../ && rm -rf libfixbuf-$FIXBUF_VERSION
+
+FROM debian:11-slim
+LABEL maintainer="maheckathorn@cert.org"
+
+RUN mkdir /netsa
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        libssl1.1 \
+        pkg-config \
+        libglib2.0-0 \
+        && apt-get clean && \
+        rm -rf /var/lib/apt/lists/*
+
+ENV PATH=$PATH:/netsa/bin
+
+COPY --from=build /netsa/ /netsa/
diff --git a/LICENSE b/LICENSE
diff --git a/Makefile b/Makefile
@@ -0,0 +1,28 @@
+# source: https://jmkhael.io/makefiles-for-your-dockerfiles/
+# Run in parallel via make -j2 see: https://stackoverflow.com/a/9220818
+
+NS = cmusei
+export SOFTWARE_NAME = fixbuf
+
+export IMAGE_NAME += $(NS)/$(SOFTWARE_NAME)
+
+export WORK_DIR = .
+
+.PHONY: build build2 build3 test
+
+build: build2
+
+build2:
+	docker build --build-arg http_proxy --build-arg https_proxy --build-arg no_proxy -t $(IMAGE_NAME):latest -f Dockerfile .
+	docker tag $(IMAGE_NAME):latest $(IMAGE_NAME):2
+
+build3:
+	docker build --build-arg http_proxy --build-arg https_proxy --build-arg no_proxy --build-arg FIXBUF_VERSION=3.0.0.alpha2 -t $(IMAGE_NAME):3 -f Dockerfile .	
+
+test:
+	docker rm -f $(SOFTWARE_NAME)
+	docker run --name=$(SOFTWARE_NAME) -td $(IMAGE_NAME)
+	py.test --hosts='docker://$(SOFTWARE_NAME)'
+	docker rm -f $(SOFTWARE_NAME)
+
+default: build
diff --git a/README.md b/README.md
@@ -0,0 +1,115 @@
+[![Software Engineering Institute](https://avatars.githubusercontent.com/u/12465755?s=200&v=4)](https://www.sei.cmu.edu/)
+
+[![Blog](https://img.shields.io/static/v1.svg?color=468f8b&labelColor=555555&logoColor=ffffff&style=for-the-badge&label=SEI&message=Blog)](https://insights.sei.cmu.edu/blog/ "blog posts from our experts in Software Engineering.")
+[![Youtube](https://img.shields.io/static/v1.svg?color=468f8b&labelColor=555555&logoColor=ffffff&style=for-the-badge&label=SEI&message=Youtube&logo=youtube)](https://www.youtube.com/@TheSEICMU/ "vidoes from our experts in Software Engineering.")
+[![Podcasts](https://img.shields.io/static/v1.svg?color=468f8b&labelColor=555555&logoColor=ffffff&style=for-the-badge&label=SEI&message=Podcasts&logo=applepodcasts)](https://insights.sei.cmu.edu/podcasts/ "podcasts from our experts in Software Engineering.")
+[![GitHub](https://img.shields.io/static/v1.svg?color=468f8b&labelColor=555555&logoColor=ffffff&style=for-the-badge&label=SEI&message=GitHub&logo=github)](https://github.com/cmu-sei "view the source for all of our repositories.")
+[![Flow Tools](https://img.shields.io/static/v1.svg?color=468f8b&labelColor=555555&logoColor=ffffff&style=for-the-badge&label=SEI&message=Flow%20Tools)](https://tools.netsa.cert.org/ "documentation and source for all our flow collection and analysis tools.")
+
+
+At the [SEI](https://www.sei.cmu.edu/), we research software engineering, cybersecurity, and AI engineering problems; create innovative technologies; and put solutions into practice.
+
+Find us at:
+
+* [Blog](https://insights.sei.cmu.edu/blog/) - blog posts from our experts in Software Engineering.
+* [Youtube](https://www.youtube.com/@TheSEICMU/) - vidoes from our experts in Software Engineering.
+* [Podcasts](https://insights.sei.cmu.edu/podcasts/) - podcasts from our experts in Software Engineering.
+* [GitHub](https://github.com/cmu-sei) - view the source for all of our repositories.
+* [Flow Tools](https://tools.netsa.cert.org/) - documentation and source for all our flow collection and analysis tools.
+
+# [certcc/fixbuf](https://tools.netsa.cert.org/fixbuf2/index.html)
+
+
+[![CI](https://img.shields.io/github/actions/workflow/status/cmu-sei/docker-fixbuf/release.yml?style=for-the-badge&logo=github)](https://github.com/cmu-sei/docker-fixbuf/actions?query=workflow%3ARelease) [![Docker pulls](https://img.shields.io/docker/pulls/cmusei/fixbuf?color=468f8b&labelColor=555555&logoColor=ffffff&style=for-the-badge&label=pulls&logo=docker)](https://hub.docker.com/r/cmusei/fixbuf/)
+
+[libfixbuf](https://tools.netsa.cert.org/fixbuf2/index.html) is a compliant implementation of the IPFIX Protocol, as defined in the "Specification of the IPFIX Protocol for the Exchange of Flow Information" (RFC 7011). It supports the information model defined in "Information Model for IP Flow Information Export" (RFC 7012), extended as proposed by "Bidirectional Flow Export using IPFIX" (RFC 5103) to support information elements for representing biflows. libfixbuf supports structured data elements as described in "Export of Structured Data in IPFIX" (RFC 6313), which adds the ability to export basicLists, subTemplateLists, and subTemplateMultiLists. libfixbuf can export type information for IPFIX elements as described in "Exporting Type Information for IPFIX Information Elements" (RFC 5610), and it supports reading this information.
+
+## Documentation
+
+More information [here](https://tools.netsa.cert.org/fixbuf2/libfixbuf/index.html).
+
+## Usage
+
+The intention of this container is to be used as a base for building other [NetSA tool](https://tools.netsa.cert.org/index.html) container images.
+
+However, [ipfixDump](https://tools.netsa.cert.org/fixbuf2/ipfixDump.html) is included as part of the fixbuf install and can be used to print contents of an IPFIX file as human-readable text.  For example, the below command mounts the `$PWD/test` folder from the host to `/tmp` in the `fixbuf` container.  It contains a yaf produced ipfix file named `flows.yaf-20231026175944-00000.yaf`, that we use ipfixDump to produce stats for:
+
+```bash
+docker run --rm -it -v $PWD/test:/tmp \
+  cmusei/fixbuf:latest \
+  ipfixDump \
+  --in /tmp/flows.yaf-20231026175944-00000.yaf \
+  --stats \
+  --yaf
+```
+```
+*** File Stats: 2 Messages, 9 Data Records, 43 Template Records ***
+  Template ID | Records
+45873 (0xb331)| 9
+49155 (0xc003)| 0
+49156 (0xc004)| 0
+49157 (0xc005)| 0
+49160 (0xc008)| 0
+49161 (0xc009)| 0
+49171 (0xc013)| 0
+49173 (0xc015)| 0
+49176 (0xc018)| 0
+49664 (0xc200)| 0
+49670 (0xc206)| 0
+49920 (0xc300)| 0
+50176 (0xc400)| 0
+50432 (0xc500)| 0
+50688 (0xc600)| 0
+50944 (0xc700)| 0
+51200 (0xc800)| 0
+51456 (0xc900)| 0
+51712 (0xca00)| 0
+51722 (0xca0a)| 4
+51723 (0xca0b)| 4
+51969 (0xcb01)| 0
+51970 (0xcb02)| 0
+51971 (0xcb03)| 0
+52224 (0xcc00)| 1
+52480 (0xcd00)| 0
+52736 (0xce00)| 0
+52737 (0xce01)| 0
+52738 (0xce02)| 0
+52739 (0xce03)| 0
+52740 (0xce04)| 0
+52741 (0xce05)| 0
+52742 (0xce06)| 0
+52743 (0xce07)| 0
+52744 (0xce08)| 0
+52745 (0xce09)| 0
+52748 (0xce0c)| 0
+52749 (0xce0d)| 0
+52756 (0xce14)| 0
+52992 (0xcf00)| 0
+53251 (0xd003)| 0
+53252 (0xd004)| 0
+53253 (0xd005)| 0
+```
+
+Version 3+ of the fixbuf container contains the [ipfix2json](https://tools.netsa.cert.org/fixbuf/ipfix2json.html) tool, which can be used to print contents of an IPFIX file as json text.  For example, using the same IPFIX input file from above `flows.yaf-20231026175944-00000.yaf`:
+
+```bash
+docker run --rm -it -v $PWD/test:/tmp \
+  cmusei/fixbuf:3 \
+  ipfix2json \
+  --in /tmp/flows.yaf-20231026175944-00000.yaf \
+  --out -
+```
+```
+{"template_record:0xd005()":["certToolId","observationTimeSeconds"]}
+{"template_record:0xc003()":["tcpSequenceNumber","initialTCPFlags","unionTCPFlags"]}
+{"template_record:0xce07()":["dnsTXTData"]}
+{"template_record:0xc009()":["mptcpInitialDataSequenceNumber","mptcpReceiverToken","mptcpMaximumSegmentSize","mptcpAddressId","mptcpFlags"]}
+{"template_record:0xca00()":["basicList","basicList","basicList","basicList","basicList","basicList","basicList"]}
+{"template_record:0xc400()":["tftpFilename","tftpMode"]}
+{"template_record:0xce02()":["sourceIPv6Address"]}
+{"template_record:0xce0d()":["mysqlCommandText","mysqlCommandCode","paddingOctets"]}
+{"template_record:0xc004()":["sourceMacAddress","destinationMacAddress"]}
+{"template_record:0xce08()":["dnsSRVTarget","dnsSRVPriority","dnsSRVWeight","dnsSRVPort","paddingOctets"]}
+{"template_record:0xc015()":["dataByteCount","averageInterarrivalTime","standardDeviationInterarrivalTime","tcpUrgTotalCount","smallPacketCount","nonEmptyPacketCount","largePacketCount","firstNonEmptyPacketSize","maxPacketSize","standardDeviationPayloadLength","firstEightNonEmptyPacketDirections","paddingOctets","reverseDataByteCount","reverseAverageInterarrivalTime","reverseStandardDeviationInterarrivalTime","reverseTcpUrgTotalCount","reverseSmallPacketCount","reverseNonEmptyPacketCount","reverseLargePacketCount","reverseFirstNonEmptyPacketSize","reverseMaxPacketSize","reverseStandardDeviationPayloadLength","paddingOctets"]}
+snip...  
+```
diff --git a/tests/default/test_default.py b/tests/default/test_default.py
@@ -0,0 +1,8 @@
+def test_fixbuf_version(host):
+    version = "2.5.0"
+    command = """PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/netsa/lib/pkgconfig \
+                 pkg-config --modversion libfixbuf"""
+
+    cmd = host.run(command)
+
+    assert version in cmd.stdout

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+__pycache__/`
	`2`	`+inventory`
	`3`	`+pytest_junit.xml`