Merge branch 'main' into kafka-integration

Signed-off-by: parmesant <[email protected]>

Author: parmesant

.github/workflows/release.yml

@@ -37,11 +37,11 @@ jobs:
         with:
           use-cross: true
           command: build
-          args: --release --target ${{matrix.target}}
+          args: --profile release-lto --target ${{matrix.target}}
 
       - name: Rename binary
         run: |
-          mv target/${{ matrix.target }}/release/parseable Parseable_OSS_${{ matrix.target }}
+          mv target/${{ matrix.target }}/release-lto/parseable Parseable_OSS_${{ matrix.target }}
 
       - name: Generate artifact attestation
         uses: actions/attest-build-provenance@v1
@@ -82,11 +82,11 @@ jobs:
           override: true
 
       - name: Build
-        run: cargo build --all --release --target x86_64-pc-windows-msvc
+        run: cargo build --all --profile release-lto --target x86_64-pc-windows-msvc
 
       - name: Rename binary
         run: |
-          mv target/x86_64-pc-windows-msvc/release/PARSEABLE.exe Parseable_OSS_x86_64-pc-windows-msvc.exe
+          mv target/x86_64-pc-windows-msvc/release-lto/PARSEABLE.exe Parseable_OSS_x86_64-pc-windows-msvc.exe
 
       - name: Generate artifact attestation
         uses: actions/attest-build-provenance@v1
@@ -135,12 +135,12 @@ jobs:
 
       - name: Build
         run: |
-          cargo build --release --target ${{ matrix.target }}
-          strip target/${{ matrix.target }}/release/Parseable
+          cargo build --profile release-lto --target ${{ matrix.target }}
+          strip target/${{ matrix.target }}/release-lto/Parseable
 
       - name: Rename binary
         run: |
-          mv target/${{ matrix.target }}/release/Parseable Parseable_OSS_${{ matrix.target }}
+          mv target/${{ matrix.target }}/release-lto/Parseable Parseable_OSS_${{ matrix.target }}
 
       - name: Generate artifact attestation
         uses: actions/attest-build-provenance@v1

Cargo.toml

@@ -128,3 +128,8 @@ assets-sha1 = "9d5a45f204d709a2dd96f6a5e0b21b3834ee0e36"
 
 [features]
 debug = []
+
+[profile.release-lto]
+inherits = "release"
+lto = "fat"
+codegen-units = 1

README.md

@@ -2,67 +2,49 @@
     <picture>
       <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/parseablehq/.github/main/images/logo-dark.png">
       <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/parseablehq/.github/main/images/logo.svg">
-      <a href="https://www.parseable.io" target="_blank"><img src="https://raw.githubusercontent.com/parseablehq/.github/main/images/logo.svg" alt="Parseable logo" /></a>
+      <a href="https://www.parseable.com" target="_blank"><img src="https://raw.githubusercontent.com/parseablehq/.github/main/images/logo.svg" alt="Parseable logo" /></a>
     </picture>
-    <br>
-    The <i>versatile</i> log database for search, analysis, and monitoring
 </h2>
 
 <div align="center">
 
 [![Docker Pulls](https://img.shields.io/docker/pulls/parseable/parseable?logo=docker&label=Docker%20Pulls)](https://hub.docker.com/r/parseable/parseable)
 [![Slack](https://img.shields.io/badge/slack-brightgreen.svg?logo=slack&label=Community&style=flat&color=%2373DC8C&)](https://logg.ing/community)
-[![Docs](https://img.shields.io/badge/stable%20docs-parseable.io%2Fdocs-brightgreen?style=flat&color=%2373DC8C&label=Docs)](https://logg.ing/docs)
+[![Docs](https://img.shields.io/badge/stable%20docs-parseable.com%2Fdocs-brightgreen?style=flat&color=%2373DC8C&label=Docs)](https://logg.ing/docs)
 [![Build](https://img.shields.io/github/checks-status/parseablehq/parseable/main?style=flat&color=%2373DC8C&label=Checks)](https://github.com/parseablehq/parseable/actions)
 
-[Key Concepts](https://www.parseable.io/docs/concepts) | [Features](https://github.com/parseablehq/parseable#rocket-highlights) | [Documentation](https://www.parseable.io/docs) | [Demo](https://demo.parseable.com/login?q=eyJ1c2VybmFtZSI6ImFkbWluIiwicGFzc3dvcmQiOiJhZG1pbiJ9) | [Integrations](https://www.parseable.io/docs/category/integrations) | [FAQ](https://www.parseable.io/docs/faq)
+[Key Concepts](https://www.parseable.com/docs/concepts) | [Features](https://github.com/parseablehq/parseable#rocket-highlights) | [Documentation](https://www.parseable.com/docs) | [Demo](https://demo.parseable.com/login?q=eyJ1c2VybmFtZSI6ImFkbWluIiwicGFzc3dvcmQiOiJhZG1pbiJ9) | [FAQ](https://www.parseable.com/docs/faq)
 
 </div>
 
-Parseable is a **cloud native log analytics platform, with a focus on simplicity & resource efficiency**. Parseable is useful for use cases where **complete data ownership, security and privacy are paramount**.
+Parseable is a disk **_less_**, cloud native database for logs, observability, security, and compliance. Parseable is built with focus on simplicity & resource efficiency. Parseable is useful for use cases where **complete data ownership, data security and privacy are paramount**.
 
-To experience Parseable UI, checkout [demo.parseable.com ↗︎](https://demo.parseable.com/login?q=eyJ1c2VybmFtZSI6ImFkbWluIiwicGFzc3dvcmQiOiJhZG1pbiJ9). You can also view the [demo video ↗︎](https://www.parseable.com/video.mp4).
+To experience Parseable UI, checkout [demo.parseable.com ↗︎](https://demo.parseable.com/login?q=eyJ1c2VybmFtZSI6ImFkbWluIiwicGFzc3dvcmQiOiJhZG1pbiJ9).
 
-## QuickStart :zap:
+## Quickstart :zap:
+
+### Run Parseable
 
 <details>
 <summary><a href="https://www.parseable.com/docs/docker-quick-start">Docker Image</a></summary>
 <p>
 
-You can <a href="https://www.parseable.com/docs/docker-quick-start">get started with Parseable Docker</a> with a simple Docker run and then send data via cURL to understand how you can ingest data to Parseable. Below is the command to run Parseable in local storage mode with Docker.
+Get started with Parseable Docker with a single command:
 
 ```bash
 docker run -p 8000:8000 \
   containers.parseable.com/parseable/parseable:latest \
   parseable local-store
 ```
 
-Once this runs successfully, you'll see dashboard at [http://localhost:8000 ↗︎](http://localhost:8000). You can login to the dashboard default credentials `admin`, `admin`.
-
-To ingest data, run the below command. This will send logs to the `demo` stream. You can see the logs in the dashboard.
-
-```bash
-curl --location --request POST 'http://localhost:8000/api/v1/ingest' \
---header 'X-P-Stream: demo' \
---header 'Authorization: Basic YWRtaW46YWRtaW4=' \
---header 'Content-Type: application/json' \
---data-raw '[
-    {
-        "id": "434a5f5e-2f5f-11ed-a261-0242ac120002",
-        "datetime": "24/Jun/2022:14:12:15 +0000",
-        "host": "153.10.110.81"
-    }
-]'
-```
-
 </p>
 </details>
 
 <details>
 <summary><a href="https://www.parseable.com/docs/docker-quick-start">Executable Binary</a></summary>
 <p>
 
-You can download and run the Parseable binary on your laptop.
+Download and run the Parseable binary on your laptop:
 
 - Linux or MacOS
 
@@ -76,9 +58,12 @@ curl -fsSL https://logg.ing/install | bash
 powershell -c "irm https://logg.ing/install-windows | iex"
 ```
 
-Once this runs successfully, you'll see dashboard at [http://localhost:8000 ↗︎](http://localhost:8000). You can login to the dashboard default credentials `admin`, `admin`.
+</p>
+</details>
+
+### Ingestion and query
 
-To ingest data, run the below command. This will send logs to the `demo` stream. You can see the logs in the dashboard.
+Once you have Parseable running, ingest data with the below command. This will send logs to the `demo` stream. You can see the logs in the dashboard.
 
 ```bash
 curl --location --request POST 'http://localhost:8000/api/v1/ingest' \
@@ -94,16 +79,24 @@ curl --location --request POST 'http://localhost:8000/api/v1/ingest' \
 ]'
 ```
 
-</p>
-</details>
+Access the Parseable UI at [http://localhost:8000 ↗︎](http://localhost:8000). You can login to the dashboard default credentials `admin`, `admin`.
+
+## Getting started :bulb:
+
+For quickstart, refer the [quickstart section ↗︎](#quickstart-zap).
+
+This section elaborates available options to run Parseable in production or development environments.
+
+- Distributed Parseable on Kubernetes: [Helm Installation](https://www.parseable.com/docs/server/installation/distributed/setup-distributed-parseable-on-kubernetes-via-helm).
+- Distributed Parseable on AWS EC2 / VMs / Linux: [Binary Installation](https://www.parseable.com/docs/server/installation/distributed/setup-systemd-service-for-distributed-parseable-server).
 
 ## Features :rocket:
 
 - [High availability & Cluster mode ↗︎](https://www.parseable.com/docs/concepts/distributed-architecture)
 - [Local cache ↗︎](https://www.parseable.com/docs/features/tiering)
-- [Alerts ↗︎](https://www.parseable.io/docs/alerts)
-- [Role based access control ↗︎](https://www.parseable.io/docs/rbac)
-- [OAuth2 support ↗︎](https://www.parseable.io/docs/oidc)
+- [Alerts ↗︎](https://www.parseable.com/docs/alerts)
+- [Role based access control ↗︎](https://www.parseable.com/docs/rbac)
+- [OAuth2 support ↗︎](https://www.parseable.com/docs/oidc)
 - [LLM integration ↗︎](https://www.parseable.com/docs/integrations/llm-based-sql-generation)
 - [OpenTelemetry support ↗︎](https://www.parseable.com/docs/opentelemetry)
 
@@ -121,14 +114,14 @@ Traditionally, logging has been seen as a text search problem. Log volumes were
 
 But with log data growing exponentially, today's log data challenges involve whole lot more – Data ingestion, storage, and observation, all at scale. We are building Parseable to address these challenges.
 
-## Verify Image Build Provenance Attestation
-All the image builds/pushes are attested for build provenance and integrity using the [attest-build-provenance](https://github.com/actions/attest-build-provenance) action. The attestations can be verified by having the latest version of [GitHub CLI](https://github.com/cli/cli/releases/latest) installed in your system. Then, execute the following command:
+## Verify images :writing_hand:
+
+Parseable builds are attested for build provenance and integrity using the [attest-build-provenance](https://github.com/actions/attest-build-provenance) action. The attestations can be verified by having the latest version of [GitHub CLI](https://github.com/cli/cli/releases/latest) installed in your system. Then, execute the following command:
 
 ```sh
 gh attestation verify PATH/TO/YOUR/PARSEABLE/ARTIFACT-BINARY -R parseablehq/parseable
 ```
 
-
 ## Contributing :trophy:
 
 <a href="https://github.com/parseablehq/parseable/graphs/contributors"><img src="https://contrib.rocks/image?repo=parseablehq/parseable" /></a>

helm-releases/parseable-1.6.5.tgz

@@ -2,7 +2,7 @@ apiVersion: v2
 name: parseable
 description: Helm chart for Parseable Server
 type: application
-version: 1.6.4
+version: 1.6.5
 appVersion: "v1.6.3"
 maintainers:
   - name: Parseable Team

helm/Chart.yaml

@@ -42,7 +42,7 @@ spec:
         imagePullPolicy: {{ .Values.parseable.image.pullPolicy }}
         args:
           - /usr/bin/parseable
-          - {{ .Values.parseable.store }}
+          - {{ if eq .Values.parseable.store "gcs-store" }}"s3-store"{{ else }}{{ .Values.parseable.store | quote }}{{ end }}
           - --ingestor-endpoint=$(HOSTNAME).{{ include "parseable.fullname" . }}-ingestor-headless.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.parseable.highAvailability.ingestor.port }}
         env:
           {{- range $key, $value :=  .Values.parseable.highAvailability.ingestor.env }}
@@ -69,6 +69,20 @@ spec:
           {{- end }}
           {{- end }}
 
+          {{- if and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled }}
+          {{- range $secret := .Values.parseable.gcsModeSecret.secrets }}
+          {{- range $key := $secret.keys }}
+          {{- $envPrefix := $secret.prefix | default "" | upper }}
+          {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
+          - name: {{ $envPrefix }}{{ $envKey | replace "GCS" "S3"}}
+            valueFrom:
+              secretKeyRef:
+                name: {{ $secret.name }}
+                key: {{ $key }}
+          {{- end }}
+          {{- end }}
+          {{- end }}
+
           {{- if and .Values.parseable.blobModeSecret .Values.parseable.blobModeSecret.enabled }}
           {{- range $secret := .Values.parseable.blobModeSecret.secrets }}
           {{- range $key := $secret.keys }}

helm/templates/ingestor-statefulset.yaml

@@ -43,9 +43,7 @@ spec:
           {{- toYaml .Values.parseable.securityContext | nindent 8 }}
         image: {{ .Values.parseable.image.repository }}:{{ .Values.parseable.image.tag | default .Chart.AppVersion }}
         imagePullPolicy: {{ .Values.parseable.image.pullPolicy }}
-        args: 
-        - /usr/bin/parseable
-        - {{ .Values.parseable.store }}
+        args: ["/usr/bin/parseable", {{ if eq .Values.parseable.store "gcs-store" }}"s3-store"{{ else }}{{ .Values.parseable.store | quote }}{{ end }}]
         env:
           - name: HOSTNAME
             valueFrom:
@@ -73,6 +71,20 @@ spec:
           {{- end }}
           {{- end }}
 
+          {{- if and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled }}
+          {{- range $secret := .Values.parseable.gcsModeSecret.secrets }}
+          {{- range $key := $secret.keys }}
+          {{- $envPrefix := $secret.prefix | default "" | upper }}
+          {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
+          - name: {{ $envPrefix }}{{ $envKey | replace "GCS" "S3"}}
+            valueFrom:
+              secretKeyRef:
+                name: {{ $secret.name }}
+                key: {{ $key }}
+          {{- end }}
+          {{- end }}
+          {{- end }}
+          
           {{- if and .Values.parseable.blobModeSecret .Values.parseable.blobModeSecret.enabled }}
           {{- range $secret := .Values.parseable.blobModeSecret.secrets }}
           {{- range $key := $secret.keys }}

helm/templates/querier-statefulset.yaml

@@ -36,7 +36,7 @@ spec:
           imagePullPolicy: {{ .Values.parseable.image.pullPolicy }}
           # Uncomment to debug
           # command: [ "/bin/sh", "-c", "sleep 1000000" ]
-          args: ["/usr/bin/parseable", {{ .Values.parseable.store | quote }}]
+          args: ["/usr/bin/parseable", {{ if eq .Values.parseable.store "gcs-store" }}"s3-store"{{ else }}{{ .Values.parseable.store | quote }}{{ end }}]
           env:
             - name: HOSTNAME
               valueFrom:
@@ -75,6 +75,20 @@ spec:
             {{- end }}
             {{- end }}
 
+            {{- if and .Values.parseable.gcsModeSecret .Values.parseable.gcsModeSecret.enabled }}
+            {{- range $secret := .Values.parseable.gcsModeSecret.secrets }}
+            {{- range $key := $secret.keys }}
+            {{- $envPrefix := $secret.prefix | default "" | upper }}
+            {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
+            - name: {{ $envPrefix }}{{ $envKey | replace "GCS" "S3"}}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $secret.name }}
+                  key: {{ $key }}
+            {{- end }}
+            {{- end }}
+            {{- end }}
+
             {{- if and .Values.parseable.blobModeSecret .Values.parseable.blobModeSecret.enabled }}
             {{- range $secret := .Values.parseable.blobModeSecret.secrets }}
             {{- range $key := $secret.keys }}

helm/templates/standalone-deployment.yaml

@@ -3,8 +3,7 @@ parseable:
     repository: containers.parseable.com/parseable/parseable
     tag: v1.6.3
     pullPolicy: Always
-  ## object store can be local-store, s3-store or blob-store.
-  ## local needs to be false if set to object store.
+  ## object store can be local-store, s3-store, blob-store or gcs-store.
   store: local-store
   ## Set to true if you want to deploy Parseable in a HA mode (multiple ingestors)
   ## Please note that highAvailability is not supported in local mode
@@ -116,6 +115,22 @@ parseable:
           - s3.secret.key
           - s3.bucket
           - s3.region
+  gcsModeSecret:
+    enabled: false
+    secrets:
+      - name: parseable-env-secret
+        prefix: P_
+        keys:
+          - addr
+          - username
+          - password
+          - staging.dir
+          - fs.dir
+          - gcs.url
+          - gcs.access.key
+          - gcs.secret.key
+          - gcs.bucket
+          - gcs.region
   serviceAccount:
     create: true
     name: "parseable"
@@ -295,7 +310,7 @@ vector:
 # See fluent-bit helm documentation to learn more:
 # https://github.com/fluent/helm-charts/tree/main/charts/fluent-bit
 fluent-bit:
-  enabled: true
+  enabled: false
   kind: DaemonSet
   serverHost: parseable.parseable.svc.cluster.local
   replicaCount: 1