From d4f716d3c0bcba0bc06918c5096de01e6ebeab66 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Wed, 4 Jun 2025 15:36:15 +0000 Subject: [PATCH 01/40] Added new directories and files to support new quick-start structure --- docs/src/user-guide/index.md | 5 +++ .../quick-start/clp-json/main-page.md | 1 + .../quick-start/clp-text/main-page.md | 1 + docs/src/user-guide/quick-start/index.md | 34 +++++++++++++++++++ .../user-guide/quick-start/setup/main-page.md | 1 + 5 files changed, 42 insertions(+) create mode 100644 docs/src/user-guide/quick-start/clp-json/main-page.md create mode 100644 docs/src/user-guide/quick-start/clp-text/main-page.md create mode 100644 docs/src/user-guide/quick-start/index.md create mode 100644 docs/src/user-guide/quick-start/setup/main-page.md diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index 642eac4a62..18e82ddcce 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -52,6 +52,11 @@ quick-start-overview quick-start-cluster-setup/index quick-start-compression/index quick-start-search/index +quick-start/index +quick-start/setup/main-page +quick-start/clp-json/main-page +quick-start/clp-text/main-page + ::: :::{toctree} diff --git a/docs/src/user-guide/quick-start/clp-json/main-page.md b/docs/src/user-guide/quick-start/clp-json/main-page.md new file mode 100644 index 0000000000..553da97d8b --- /dev/null +++ b/docs/src/user-guide/quick-start/clp-json/main-page.md @@ -0,0 +1 @@ +# CLP-JSON diff --git a/docs/src/user-guide/quick-start/clp-text/main-page.md b/docs/src/user-guide/quick-start/clp-text/main-page.md new file mode 100644 index 0000000000..113a0acede --- /dev/null +++ b/docs/src/user-guide/quick-start/clp-text/main-page.md @@ -0,0 +1 @@ +# CLP-text diff --git a/docs/src/user-guide/quick-start/index.md b/docs/src/user-guide/quick-start/index.md new file mode 100644 index 0000000000..be4a7cccb5 --- /dev/null +++ b/docs/src/user-guide/quick-start/index.md @@ -0,0 +1,34 @@ +# Overview + +To learn how to use CLP, follow the "Setup" link. + +::::{grid} 1 1 1 1 +:gutter: 2 + +:::{grid-item-card} +:link: setup/main-page +Setup +^^^ +View system requirements and choose a flavour of CLP. +::: +:::: + +If you already have CLP set up on your computer, click your flavour of CLP below to learn how to use it. + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: clp-json/main-page +CLP for JSON logs +^^^ +Learn about compressing and searching JSON logs. +::: + +:::{grid-item-card} +:link: clp-text/main-page +CLP for unstructured logs +^^^^^^^^^^^^ +Learn about compressing and searching unstructured logs. +::: +:::: diff --git a/docs/src/user-guide/quick-start/setup/main-page.md b/docs/src/user-guide/quick-start/setup/main-page.md new file mode 100644 index 0000000000..feae8cb576 --- /dev/null +++ b/docs/src/user-guide/quick-start/setup/main-page.md @@ -0,0 +1 @@ +# Setup From 28df225c193893959da276109771a211e8775c83 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Wed, 4 Jun 2025 18:47:30 +0000 Subject: [PATCH 02/40] Add 'setup' page in Quick-start --- .../user-guide/quick-start/setup/main-page.md | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/docs/src/user-guide/quick-start/setup/main-page.md b/docs/src/user-guide/quick-start/setup/main-page.md index feae8cb576..238bff9157 100644 --- a/docs/src/user-guide/quick-start/setup/main-page.md +++ b/docs/src/user-guide/quick-start/setup/main-page.md @@ -1 +1,132 @@ # Setup + +This page will guide you through CLP's system requirements, and help you choose which flavour of CLP will be best for you. + +--- + +## System Requirements + +To run properly on your system, CLP requires a few other programs. Before you set up CLP, ensure that you have the following installed. + +### Docker + +CLP uses Docker to deploy its different components. You can check whether Docker is already installed on your system by running the command + +``` bash +docker version +``` + +If Docker is not on your system, follow the instructions [here][Docker] to install it. + +If you're not running as root, ensure Docker can be run [without superuser privileges][docker-non-root]. + +If you're using Docker Desktop, ensure version 4.34 or higher is installed, and [host networking is enabled][docker-desktop-host-networking]. + +### Python + +CLP uses Python to interpret the scripts that coordinate how it runs. Specifically, CLP needs Python version 3.8 or higher. Many Linux distributions come with Python pre-installed, but just to be sure, run the command + +``` bash +python3 --version +``` + +If Python isn't on your system, or if the version isn't high enough, follow the instructions [here][Python] to install or upgrade it. + +:::{note} +If you're planning on deploying CLP on multiple nodes/systems, there are a few other system requirements; check out the [multi-node deployment](../../quick-start-cluster-setup/multi-node) page for more details. +::: + +--- + +## Choosing a flavour + +CLP has two flavours. Each flavour deals with a different type of log. + +* **[clp-json](#clp-json)** for compressing and searching **JSON** logs. +* **[clp-text](#clp-text)** for compressing and searching **unstructured text** logs. + +:::{note} +Both flavours contain the same binaries but are configured with different values for the +`package.storage_engine` key in the package's config file (`etc/clp-config.yml`). +::: + +### clp-json + +The JSON flavour of CLP is appropriate for JSON logs where each log event is an independent JSON +object. For example: + +```json lines +{ + "t": { + "$date": "2023-03-21T23:46:37.392" + }, + "ctx": "conn11", + "msg": "Waiting for write concern." +} +{ + "t": { + "$date": "2023-03-21T23:46:37.392" + }, + "msg": "Set last op to system time" +} +``` + +The log file above contains two log events represented by two JSON objects printed one after the +other. Whitespace is ignored, so the log events could also appear with no newlines and indentation. + +If your logs are JSON logs, download and extract the JSON flavour tarball from the [Releases][clp-releases] page on GitHub, and proceed to the CLP-JSON portal below. + +::::{grid} 1 1 1 1 +:gutter: 2 + +:::{grid-item-card} +:link: ../clp-json/main-page +CLP for JSON logs +^^^ +Learn how to start up CLP-JSON, and begin compressing and searching JSON logs. +::: +:::: + +### clp-text + +The unstructured text flavour of CLP is appropriate for unstructured text logs where each log event contains a +timestamp and may span one or more lines. + +:::{note} +If your logs do not contain timestamps or CLP can't automatically parse the timestamps in your logs, +it will treat each line as an independent log event. +::: + +For example: + +``` +2015-03-23T15:50:17.926Z INFO container_1 Transitioned from ALLOCATED to ACQUIRED +2015-03-23T15:50:17.927Z ERROR Scheduler: Error trying to assign container token +java.lang.IllegalArgumentException: java.net.UnknownHostException: i-e5d112ea + at org.apache.hadoop.security.buildTokenService(SecurityUtil.java:374) + at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2033) +Caused by: java.net.UnknownHostException: i-e5d112ea + ... 17 more +``` + +The log file above contains two log events, both beginning with a timestamp. The first is a single +line while the second contains multiple lines. + +If your logs are unstructured text logs, download and extract the unstructured text flavour tarball from the [Releases][clp-releases] page on GitHub, and proceed to the CLP-text portal below. + +::::{grid} 1 1 1 1 +:gutter: 2 + +:::{grid-item-card} +:link: ../clp-text/main-page +CLP for unstructured logs +^^^ +Learn how to start up CLP-text, and begin compressing and searching unstructured text logs. +::: +:::: + +[Docker]: https://docs.docker.com/engine/install/ +[docker-non-root]: https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user +[docker-desktop-host-networking]: https://docs.docker.com/engine/network/drivers/host/#docker-desktop +[Python]: https://www.geeksforgeeks.org/how-to-install-python-on-linux/ +[clp-releases]: https://github.com/y-scope/clp/releases From 05759bc34864538d3177d38405ebc6cd08c9e41b Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Wed, 4 Jun 2025 19:44:07 +0000 Subject: [PATCH 03/40] Add clp-json main page and connect to setup page --- .../quick-start/clp-json/clp-search-ui.png | 3 + .../quick-start/clp-json/main-page.md | 146 ++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 docs/src/user-guide/quick-start/clp-json/clp-search-ui.png diff --git a/docs/src/user-guide/quick-start/clp-json/clp-search-ui.png b/docs/src/user-guide/quick-start/clp-json/clp-search-ui.png new file mode 100644 index 0000000000..74530978a8 --- /dev/null +++ b/docs/src/user-guide/quick-start/clp-json/clp-search-ui.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80e2309f9b03856672edd73ae34dde67814ca04e2cd5235586b78a67fc4f18bf +size 57858 diff --git a/docs/src/user-guide/quick-start/clp-json/main-page.md b/docs/src/user-guide/quick-start/clp-json/main-page.md index 553da97d8b..d4995bb680 100644 --- a/docs/src/user-guide/quick-start/clp-json/main-page.md +++ b/docs/src/user-guide/quick-start/clp-json/main-page.md @@ -1 +1,147 @@ # CLP-JSON + +This page will walk you through how to start up CLP, and also show you how to use CLP to compress and search JSON logs. + +:::{caution} +If you're using the `clp-json` release, you can only compress JSON logs. If you're using the +`clp-text` release, you should only compress unstructured text logs (`clp-text` can compress and +search JSON logs as if it was unstructured text, but `clp-text` cannot query individual fields). +This limitation will be addressed in a future version of CLP. +::: + +--- + +## Starting CLP + +To start CLP, run + +```bash +sbin/start-clp.sh +``` + +:::{note} +If CLP fails to start (e.g., due to a port conflict), try adjusting the settings in +`etc/clp-config.yml` and then run the start command again. +::: + +--- + +## Compressing JSON logs + +To compress JSON logs, from inside the package directory, run: + +```bash +sbin/compress.sh fs --timestamp-key '' [ ...] +``` + +* `` is the field path of the kv-pair that contains the timestamp in each log event. + * E.g., if your log events look like + `{"timestamp": {"iso8601": "2024-01-01 00:01:02.345", ...}}`, you should enter + `timestamp.iso8601` as the timestamp key. + + :::{caution} + Log events without the specified timestamp key will _not_ have an assigned timestamp. Currently, + these events can only be searched from the command line (when you don't specify a timestamp + filter). + ::: + +* `` are paths to JSON log files or directories containing such files. + * Each JSON log file should contain each log event as a separate JSON object, + i.e., _not_ as an array. + +Compressed logs will be stored in the `/var/bin/archives` directory. + +:::{tip} +To compress logs from object storage, see +[Using object storage](../../guides-using-object-storage/index). +::: + +### Sample logs + +For some sample logs, check out the open-source [datasets](../../resources-datasets.md). + +### Examining compression statistics + +The compression script used above will output the compression ratio of each dataset you compress, or +you can use the UI to view overall statistics. + +--- + +## Searching JSON logs + +You can search through your logs using queries from the UI or from the command line. + +### Queries + +Regardless of what method you use to search, you'll need a query to find the logs you're looking for. Queries for JSON logs take the general form of + +```bash +key: value +``` + +where `key` is the name of the JSON key you'd like to search within, and `value` is a sequence of characters you're looking for within that key. Multiple key-value pairs can be chained together with `AND`, `OR`, or `NOT`, like so: + +```bash +key1: value1 AND key2: value2 OR key3: value3 ... +``` + +There are a number of other JSON-specific syntax rules that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference-json-search-syntax). + +### Searching from the command line + +If you'd like to search your query from the command line, run the following command from inside the package: + +```bash +sbin/search.sh '' +``` + +To narrow your search to a specific time range: + +* Add `--begin-time ` to filter for log events after a certain time. + * `` is the timestamp as milliseconds since the UNIX epoch. +* Add `--end-time ` to filter for log events after a certain time. + +To perform case-insensitive searches, add the `--ignore-case` flag. + +:::{caution} +To match the convention of other tools, by default, searches are case-**insensitive** in the UI and +searches are case-**sensitive** on the command line. +::: + +### Searching from the UI + +If you'd like to search your query from the web UI, CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). + +:::{image} clp-search-ui.png +::: + +The image above shows the search page after running a query. The numbered circles correspond to the following features: + +1. The search box is where you can enter your query. +2. The timeline shows the number of results across the time range of your query. + * You can click and drag to zoom into a time range or use the time range filter in (4). +3. The table displays the search results for your query. +4. Clicking the icon reveals additional filters for your query. + * The time range filter allows you to specify the period of time that matching log events must be + in. + * The case sensitivity filter allows you to specify whether CLP should respect the case of your + query. +5. Clicking the icon reveals options for displaying results. +6. The icon clears the results of the last query. + +:::{note} +By default, the UI will only return 1,000 of the latest search results. To perform searches which +return more results, use the [command line](#searching-from-the-command-line). +::: + +--- + +## Stopping CLP + +If you need to stop CLP, run the command + +```bash +sbin/stop-clp.sh +``` + +[json-log-format]: ../quick-start-cluster-setup/index.md#clp-json From 0eaa51646e3225b6eca8a03f2f820d7005258c07 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 5 Jun 2025 14:23:34 +0000 Subject: [PATCH 04/40] Add clp-text main page and connect to setup page --- .../quick-start/clp-text/clp-search-ui.png | 3 + .../quick-start/clp-text/main-page.md | 127 ++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 docs/src/user-guide/quick-start/clp-text/clp-search-ui.png diff --git a/docs/src/user-guide/quick-start/clp-text/clp-search-ui.png b/docs/src/user-guide/quick-start/clp-text/clp-search-ui.png new file mode 100644 index 0000000000..74530978a8 --- /dev/null +++ b/docs/src/user-guide/quick-start/clp-text/clp-search-ui.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80e2309f9b03856672edd73ae34dde67814ca04e2cd5235586b78a67fc4f18bf +size 57858 diff --git a/docs/src/user-guide/quick-start/clp-text/main-page.md b/docs/src/user-guide/quick-start/clp-text/main-page.md index 113a0acede..af8eb9421f 100644 --- a/docs/src/user-guide/quick-start/clp-text/main-page.md +++ b/docs/src/user-guide/quick-start/clp-text/main-page.md @@ -1 +1,128 @@ # CLP-text + +This page will walk you through how to start up CLP, and also show you how to use CLP to compress and search unstructured text logs. + +:::{caution} +If you're using the `clp-json` release, you can only compress JSON logs. If you're using the +`clp-text` release, you should only compress unstructured text logs (`clp-text` can compress and +search JSON logs as if it was unstructured text, but `clp-text` cannot query individual fields). +This limitation will be addressed in a future version of CLP. +::: + +--- + +## Starting CLP + +To start CLP, run + +```bash +sbin/start-clp.sh +``` + +:::{note} +If CLP fails to start (e.g., due to a port conflict), try adjusting the settings in +`etc/clp-config.yml` and then run the start command again. +::: + +--- + +## Compressing unstructured text logs + +To compress unstructured text logs, run the following from inside the package directory: + +```bash +sbin/compress.sh [ ...] +``` + +`` are paths to unstructured text log files or directories containing such files. + +Compressed logs will be stored in the `/var/bin/archives` directory. + +# Sample logs + +For some sample logs, check out the open-source [datasets](../../resources-datasets.md). + +# Examining compression statistics + +The compression script used above will output the compression ratio of each dataset you compress, or +you can use the UI to view overall statistics. + +--- + +## Searching unstructured text logs + +You can search through your logs using queries from the UI or from the command line. + +### Queries + +Regardless of what method you use to search, you'll need a query to find the logs you're looking for. All unstructured text log queries are written as plain text. + +You can use a couple special characters to make these queries more versatile. `*` can be used as a placeholder for an unknown number of characters, and `?` can be used for a single character. For example, the query + +``` bash +one * two?three +``` + +would return all logs that contain the sequence of characters "one" followed by any number (even zero) of other characters, followed by "two", followed by one other character, followed by "three". + + + + +There are a number of other syntax rules specific to unstructured text queries that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference-text-search-syntax). + +### Searching from the command line + +If you'd like to search your query from the command line, run the following command from inside the package: + +```bash +sbin/search.sh '' +``` + +To narrow your search to a specific time range: + +* Add `--begin-time ` to filter for log events after a certain time. + * `` is the timestamp as milliseconds since the UNIX epoch. +* Add `--end-time ` to filter for log events after a certain time. + +To perform case-insensitive searches, add the `--ignore-case` flag. + +:::{caution} +To match the convention of other tools, by default, searches are case-**insensitive** in the UI and +searches are case-**sensitive** on the command line. +::: + +### Searching from the UI + +If you'd like to search your query from the web UI, CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). + +:::{image} clp-search-ui.png +::: + +The image above shows the search page after running a query. The numbered circles correspond to the following features: + +1. The search box is where you can enter your query. +2. The timeline shows the number of results across the time range of your query. + * You can click and drag to zoom into a time range or use the time range filter in (4). +3. The table displays the search results for your query. +4. Clicking the icon reveals additional filters for your query. + * The time range filter allows you to specify the period of time that matching log events must be + in. + * The case sensitivity filter allows you to specify whether CLP should respect the case of your + query. +5. Clicking the icon reveals options for displaying results. +6. The icon clears the results of the last query. + +:::{note} +By default, the UI will only return 1,000 of the latest search results. To perform searches which +return more results, use the [command line](#searching-from-the-command-line). +::: + +--- + +## Stopping CLP + +If you need to stop CLP, run the command + +```bash +sbin/stop-clp.sh +``` From 83e1c984fa47d96afd8dca218532f2d96d9fe3c6 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 5 Jun 2025 14:29:02 +0000 Subject: [PATCH 05/40] Modify caution message in clp-json and clp-text --- docs/src/user-guide/quick-start/clp-json/main-page.md | 7 +------ docs/src/user-guide/quick-start/clp-text/main-page.md | 5 +---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/src/user-guide/quick-start/clp-json/main-page.md b/docs/src/user-guide/quick-start/clp-json/main-page.md index d4995bb680..0307929492 100644 --- a/docs/src/user-guide/quick-start/clp-json/main-page.md +++ b/docs/src/user-guide/quick-start/clp-json/main-page.md @@ -3,10 +3,7 @@ This page will walk you through how to start up CLP, and also show you how to use CLP to compress and search JSON logs. :::{caution} -If you're using the `clp-json` release, you can only compress JSON logs. If you're using the -`clp-text` release, you should only compress unstructured text logs (`clp-text` can compress and -search JSON logs as if it was unstructured text, but `clp-text` cannot query individual fields). -This limitation will be addressed in a future version of CLP. +If you're using the `clp-json` release, you can only compress JSON logs. ::: --- @@ -143,5 +140,3 @@ If you need to stop CLP, run the command ```bash sbin/stop-clp.sh ``` - -[json-log-format]: ../quick-start-cluster-setup/index.md#clp-json diff --git a/docs/src/user-guide/quick-start/clp-text/main-page.md b/docs/src/user-guide/quick-start/clp-text/main-page.md index af8eb9421f..a1e6efac36 100644 --- a/docs/src/user-guide/quick-start/clp-text/main-page.md +++ b/docs/src/user-guide/quick-start/clp-text/main-page.md @@ -3,10 +3,7 @@ This page will walk you through how to start up CLP, and also show you how to use CLP to compress and search unstructured text logs. :::{caution} -If you're using the `clp-json` release, you can only compress JSON logs. If you're using the -`clp-text` release, you should only compress unstructured text logs (`clp-text` can compress and -search JSON logs as if it was unstructured text, but `clp-text` cannot query individual fields). -This limitation will be addressed in a future version of CLP. +If you're using the `clp-text` release, you should only compress unstructured text logs. `clp-text` can compress and search JSON logs as if it was unstructured text, but `clp-text` cannot query individual fields. This limitation will be addressed in a future version of CLP. ::: --- From 2bd5ff89eba25925d919a1d12344d75bae0b7620 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 5 Jun 2025 16:03:35 +0000 Subject: [PATCH 06/40] Move multi-node development page to Guides; add Guides directory to accomodate --- .../guides-multi-node}/multi-node.md | 21 +++++++++++++++++-- .../{ => guides}/guides-overview.md | 7 +++++++ .../guides-using-object-storage/clp-config.md | 0 .../guides-using-object-storage/clp-usage.md | 2 +- .../guides-using-object-storage/index.md | 6 +++--- .../object-storage-config.md | 0 docs/src/user-guide/index.md | 8 ++++--- .../quick-start-cluster-setup/index.md | 1 - .../quick-start-compression/json.md | 2 +- .../quick-start/clp-json/main-page.md | 2 +- .../user-guide/quick-start/setup/main-page.md | 2 +- 11 files changed, 38 insertions(+), 13 deletions(-) rename docs/src/user-guide/{quick-start-cluster-setup => guides/guides-multi-node}/multi-node.md (93%) rename docs/src/user-guide/{ => guides}/guides-overview.md (66%) rename docs/src/user-guide/{ => guides}/guides-using-object-storage/clp-config.md (100%) rename docs/src/user-guide/{ => guides}/guides-using-object-storage/clp-usage.md (94%) rename docs/src/user-guide/{ => guides}/guides-using-object-storage/index.md (96%) rename docs/src/user-guide/{ => guides}/guides-using-object-storage/object-storage-config.md (100%) diff --git a/docs/src/user-guide/quick-start-cluster-setup/multi-node.md b/docs/src/user-guide/guides/guides-multi-node/multi-node.md similarity index 93% rename from docs/src/user-guide/quick-start-cluster-setup/multi-node.md rename to docs/src/user-guide/guides/guides-multi-node/multi-node.md index 705f2f97a6..461b7b49ee 100644 --- a/docs/src/user-guide/quick-start-cluster-setup/multi-node.md +++ b/docs/src/user-guide/guides/guides-multi-node/multi-node.md @@ -110,8 +110,25 @@ Where `` is the name of the component in the groups above. ## Using CLP -Check out the [compression](../quick-start-compression/index) and -[search](../quick-start-search/index) guides to compress and search your logs. +Check out the quick-start guide for your chosen flavour of CLP to learn how to compress and search your logs. + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: ../../../quick-start/clp-json/main-page +CLP-JSON +^^^ +Compress and search JSON logs. +::: + +:::{grid-item-card} +:link: ../../../quick-start/clp-text/main-page +CLP-text +^^^ +Compress and search unstructured text logs. +::: +:::: ## Stopping CLP diff --git a/docs/src/user-guide/guides-overview.md b/docs/src/user-guide/guides/guides-overview.md similarity index 66% rename from docs/src/user-guide/guides-overview.md rename to docs/src/user-guide/guides/guides-overview.md index 5e8179bf71..c04e1dedc8 100644 --- a/docs/src/user-guide/guides-overview.md +++ b/docs/src/user-guide/guides/guides-overview.md @@ -11,4 +11,11 @@ Using object storage ^^^ Using CLP to ingest logs from object storage and store archives on object storage. ::: + +:::{grid-item-card} +:link: guides-multi-node/multi-node +Multi-node deployment +^^^ +Learn how to deploy CLP across multiple systems/nodes. +::: :::: diff --git a/docs/src/user-guide/guides-using-object-storage/clp-config.md b/docs/src/user-guide/guides/guides-using-object-storage/clp-config.md similarity index 100% rename from docs/src/user-guide/guides-using-object-storage/clp-config.md rename to docs/src/user-guide/guides/guides-using-object-storage/clp-config.md diff --git a/docs/src/user-guide/guides-using-object-storage/clp-usage.md b/docs/src/user-guide/guides/guides-using-object-storage/clp-usage.md similarity index 94% rename from docs/src/user-guide/guides-using-object-storage/clp-usage.md rename to docs/src/user-guide/guides/guides-using-object-storage/clp-usage.md index e2d552e78f..54f6aa7fdb 100644 --- a/docs/src/user-guide/guides-using-object-storage/clp-usage.md +++ b/docs/src/user-guide/guides/guides-using-object-storage/clp-usage.md @@ -1,7 +1,7 @@ # Using CLP with object storage To compress logs from S3, follow the steps in the section below. For all other operations, you -should be able to use CLP as described in the [quick start](../quick-start-overview.md) guide. +should be able to use CLP as described in the [clp-json quick start guide](../../quick-start/clp-json/main-page). ## Compressing logs from S3 diff --git a/docs/src/user-guide/guides-using-object-storage/index.md b/docs/src/user-guide/guides/guides-using-object-storage/index.md similarity index 96% rename from docs/src/user-guide/guides-using-object-storage/index.md rename to docs/src/user-guide/guides/guides-using-object-storage/index.md index ea553ef806..7db66516eb 100644 --- a/docs/src/user-guide/guides-using-object-storage/index.md +++ b/docs/src/user-guide/guides/guides-using-object-storage/index.md @@ -11,7 +11,7 @@ to use object storage for any combination of the three use cases (e.g., compress cache the stream files on S3, but store archives on the local filesystem). :::{note} -Currently, only the [clp-json][release-choices] release supports object storage. Support for +Currently, only the [clp-json][release-choices] flavour supports object storage. Support for `clp-text` will be added in a future release. ::: @@ -23,7 +23,7 @@ will be added in a future release. ## Prerequisites 1. This guide assumes you're able to configure, start, stop, and use a CLP cluster as described in - the [quick-start guide](../quick-start-overview.md). + the [clp-json quick-start guide][release-choices]. 2. An S3 bucket and [key prefix][aws-key-prefixes] containing the logs you wish to compress. 3. An S3 bucket and key prefix where you wish to store compressed archives. 4. An S3 bucket and key prefix where you wish to cache stream files. @@ -136,4 +136,4 @@ clp-usage [aws-iam-roles]: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html [aws-key-prefixes]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html [aws-sts-credentials]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html -[release-choices]: ../quick-start-cluster-setup/index.md#choosing-a-release +[release-choices]: ../../quick-start/clp-json/main-page diff --git a/docs/src/user-guide/guides-using-object-storage/object-storage-config.md b/docs/src/user-guide/guides/guides-using-object-storage/object-storage-config.md similarity index 100% rename from docs/src/user-guide/guides-using-object-storage/object-storage-config.md rename to docs/src/user-guide/guides/guides-using-object-storage/object-storage-config.md diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index 18e82ddcce..de8d65d873 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -16,7 +16,7 @@ A quick start guide for setting up a CLP cluster, compressing your logs, and sea ::: :::{grid-item-card} -:link: guides-overview +:link: guides/guides-overview Guides ^^^ Guides for using CLP in a variety of use cases. @@ -64,8 +64,10 @@ quick-start/clp-text/main-page :caption: Guides :glob: -guides-overview -guides-using-object-storage/index +guides/guides-overview +guides/guides-using-object-storage/index +guides/guides-multi-node/multi-node + ::: :::{toctree} diff --git a/docs/src/user-guide/quick-start-cluster-setup/index.md b/docs/src/user-guide/quick-start-cluster-setup/index.md index b3042c7a9c..822cc14f11 100644 --- a/docs/src/user-guide/quick-start-cluster-setup/index.md +++ b/docs/src/user-guide/quick-start-cluster-setup/index.md @@ -94,7 +94,6 @@ Multi-node deployment :caption: Cluster setup single-node -multi-node ::: [clp-releases]: https://github.com/y-scope/clp/releases diff --git a/docs/src/user-guide/quick-start-compression/json.md b/docs/src/user-guide/quick-start-compression/json.md index 7784a4f509..63a5ab62d6 100644 --- a/docs/src/user-guide/quick-start-compression/json.md +++ b/docs/src/user-guide/quick-start-compression/json.md @@ -23,7 +23,7 @@ sbin/compress.sh --timestamp-key '' [ ...] :::{tip} To compress logs from object storage, see -[Using object storage](../guides-using-object-storage/index). +[Using object storage](../guides/guides-using-object-storage/index). ::: # Sample logs diff --git a/docs/src/user-guide/quick-start/clp-json/main-page.md b/docs/src/user-guide/quick-start/clp-json/main-page.md index 0307929492..ea596f09da 100644 --- a/docs/src/user-guide/quick-start/clp-json/main-page.md +++ b/docs/src/user-guide/quick-start/clp-json/main-page.md @@ -50,7 +50,7 @@ Compressed logs will be stored in the `/var/bin/archives` directory. :::{tip} To compress logs from object storage, see -[Using object storage](../../guides-using-object-storage/index). +[Using object storage](../../guides/guides-using-object-storage/index). ::: ### Sample logs diff --git a/docs/src/user-guide/quick-start/setup/main-page.md b/docs/src/user-guide/quick-start/setup/main-page.md index 238bff9157..bef5edc331 100644 --- a/docs/src/user-guide/quick-start/setup/main-page.md +++ b/docs/src/user-guide/quick-start/setup/main-page.md @@ -33,7 +33,7 @@ python3 --version If Python isn't on your system, or if the version isn't high enough, follow the instructions [here][Python] to install or upgrade it. :::{note} -If you're planning on deploying CLP on multiple nodes/systems, there are a few other system requirements; check out the [multi-node deployment](../../quick-start-cluster-setup/multi-node) page for more details. +If you're planning on deploying CLP on multiple nodes/systems, there are a few other system requirements; check out the [multi-node deployment](../../guides/guides-multi-node/multi-node) page for more details. ::: --- From e5be2a3e984ff63c4509e9d3969da0f324c6aeab Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 5 Jun 2025 16:35:12 +0000 Subject: [PATCH 07/40] Removed obsolete quick-start docs; the information has been integrated into the new quick-start docs --- docs/src/user-guide/index.md | 8 +- .../quick-start-cluster-setup/index.md | 99 ------------------- .../quick-start-cluster-setup/single-node.md | 40 -------- .../quick-start-compression/index.md | 33 ------- .../quick-start-compression/json.md | 38 ------- .../quick-start-compression/text.md | 18 ---- docs/src/user-guide/quick-start-overview.md | 8 -- .../quick-start-search/cli-search.md | 23 ----- .../quick-start-search/clp-search-ui.png | 3 - .../user-guide/quick-start-search/index.md | 41 -------- .../quick-start-search/ui-search.md | 30 ------ .../quick-start/clp-text/main-page.md | 7 +- 12 files changed, 4 insertions(+), 344 deletions(-) delete mode 100644 docs/src/user-guide/quick-start-cluster-setup/index.md delete mode 100644 docs/src/user-guide/quick-start-cluster-setup/single-node.md delete mode 100644 docs/src/user-guide/quick-start-compression/index.md delete mode 100644 docs/src/user-guide/quick-start-compression/json.md delete mode 100644 docs/src/user-guide/quick-start-compression/text.md delete mode 100644 docs/src/user-guide/quick-start-overview.md delete mode 100644 docs/src/user-guide/quick-start-search/cli-search.md delete mode 100644 docs/src/user-guide/quick-start-search/clp-search-ui.png delete mode 100644 docs/src/user-guide/quick-start-search/index.md delete mode 100644 docs/src/user-guide/quick-start-search/ui-search.md diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index de8d65d873..59bb85bb65 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -9,10 +9,10 @@ The sections are as follows: :gutter: 2 :::{grid-item-card} -:link: quick-start-overview +:link: quick-start/index Quick start ^^^ -A quick start guide for setting up a CLP cluster, compressing your logs, and searching them. +A quick start guide for choosing a flavour of CLP, setting it up, compressing your logs, and searching them. ::: :::{grid-item-card} @@ -48,10 +48,6 @@ Reference docs like format specifications, etc. :hidden: :caption: Quick start -quick-start-overview -quick-start-cluster-setup/index -quick-start-compression/index -quick-start-search/index quick-start/index quick-start/setup/main-page quick-start/clp-json/main-page diff --git a/docs/src/user-guide/quick-start-cluster-setup/index.md b/docs/src/user-guide/quick-start-cluster-setup/index.md deleted file mode 100644 index 822cc14f11..0000000000 --- a/docs/src/user-guide/quick-start-cluster-setup/index.md +++ /dev/null @@ -1,99 +0,0 @@ -# Cluster setup - -To set up a cluster, you'll need to: - -* Choose a release. -* Choose between a single or multi-node deployment. -* Ensure you meet the requirements for running the release. -* Configure the release (if necessary). -* Start CLP. - -## Choosing a release - -There are two flavours of CLP [releases][clp-releases]: - -* **[clp-json](#clp-json)** for compressing and searching **JSON** logs. -* **[clp-text](#clp-text)** for compressing and searching **unstructured text** logs. - -:::{note} -Both flavours contain the same binaries but are configured with different values for the -`package.storage_engine` key in the package's config file (`etc/clp-config.yml`). -::: - -You should download and extract the release that's appropriate for your logs. - -### clp-json - -`clp-json` releases are appropriate for JSON logs where each log event is an independent JSON -object. For example: - -```json lines -{ - "t": { - "$date": "2023-03-21T23:46:37.392" - }, - "ctx": "conn11", - "msg": "Waiting for write concern." -} -{ - "t": { - "$date": "2023-03-21T23:46:37.392" - }, - "msg": "Set last op to system time" -} -``` - -The log file above contains two log events represented by two JSON objects printed one after the -other. Whitespace is ignored, so the log events could also appear with no newlines and indentation. - -### clp-text - -`clp-text` releases are appropriate for unstructured text logs where each log event contains a -timestamp and may span one or more lines. - -:::{note} -If your logs do not contain timestamps or CLP can't automatically parse the timestamps in your logs, -it will treat each line as an independent log event. -::: - -For example: - -``` -2015-03-23T15:50:17.926Z INFO container_1 Transitioned from ALLOCATED to ACQUIRED -2015-03-23T15:50:17.927Z ERROR Scheduler: Error trying to assign container token -java.lang.IllegalArgumentException: java.net.UnknownHostException: i-e5d112ea - at org.apache.hadoop.security.buildTokenService(SecurityUtil.java:374) - at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2033) -Caused by: java.net.UnknownHostException: i-e5d112ea - ... 17 more -``` - -The log file above contains two log events, both beginning with a timestamp. The first is a single -line while the second contains multiple lines. - -## Deployment options - -Choose one of the deployment options below: - -::::{grid} 1 1 2 2 -:gutter: 2 - -:::{grid-item-card} -:link: single-node -Single-node deployment -::: - -:::{grid-item-card} -:link: multi-node -Multi-node deployment -::: -:::: - -:::{toctree} -:hidden: -:caption: Cluster setup - -single-node -::: - -[clp-releases]: https://github.com/y-scope/clp/releases diff --git a/docs/src/user-guide/quick-start-cluster-setup/single-node.md b/docs/src/user-guide/quick-start-cluster-setup/single-node.md deleted file mode 100644 index ea234858ac..0000000000 --- a/docs/src/user-guide/quick-start-cluster-setup/single-node.md +++ /dev/null @@ -1,40 +0,0 @@ -# Single-node deployment - -A single-node deployment allows you to run CLP on a single host. - -## Requirements - -* [Docker] - * If you're not running as root, ensure `docker` can be run - [without superuser privileges][docker-non-root]. - * If you're using Docker Desktop, ensure version 4.34 or higher is installed, and - [host networking is enabled][docker-desktop-host-networking]. -* Python 3.8 or higher - -## Starting CLP - -```bash -sbin/start-clp.sh -``` - -:::{note} -If CLP fails to start (e.g., due to a port conflict), try adjusting the settings in -`etc/clp-config.yml` and then run the start command again. -::: - -## Using CLP - -Check out the [compression](../quick-start-compression/index) and -[search](../quick-start-search/index) guides to compress and search your logs. - -## Stopping CLP - -If you need to stop the cluster, run: - -```bash -sbin/stop-clp.sh -``` - -[Docker]: https://docs.docker.com/engine/install/ -[docker-desktop-host-networking]: https://docs.docker.com/engine/network/drivers/host/#docker-desktop -[docker-non-root]: https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user diff --git a/docs/src/user-guide/quick-start-compression/index.md b/docs/src/user-guide/quick-start-compression/index.md deleted file mode 100644 index fab8a8781c..0000000000 --- a/docs/src/user-guide/quick-start-compression/index.md +++ /dev/null @@ -1,33 +0,0 @@ -# Compression - -You can compress your logs using a script in the CLP package. Depending on the format of your logs, -choose one of the options below. - -:::{caution} -If you're using the `clp-json` release, you can only compress JSON logs. If you're using the -`clp-text` release, you should only compress unstructured text logs (`clp-text` can compress and -search JSON logs as if it was unstructured text, but `clp-text` cannot query individual fields). -This limitation will be addressed in a future version of CLP. -::: - -::::{grid} 1 1 2 2 -:gutter: 2 - -:::{grid-item-card} -:link: json -Compressing JSON logs -::: - -:::{grid-item-card} -:link: text -Compressing unstructured text logs -::: -:::: - -:::{toctree} -:hidden: -:caption: Compression - -json -text -::: diff --git a/docs/src/user-guide/quick-start-compression/json.md b/docs/src/user-guide/quick-start-compression/json.md deleted file mode 100644 index 63a5ab62d6..0000000000 --- a/docs/src/user-guide/quick-start-compression/json.md +++ /dev/null @@ -1,38 +0,0 @@ -# Compressing JSON logs - -To compress JSON logs, from inside the package directory, run: - -```bash -sbin/compress.sh --timestamp-key '' [ ...] -``` - -* `` is the field path of the kv-pair that contains the timestamp in each log event. - * E.g., if your log events look like - `{"timestamp": {"iso8601": "2024-01-01 00:01:02.345", ...}}`, you should enter - `timestamp.iso8601` as the timestamp key. - - :::{caution} - Log events without the specified timestamp key will _not_ have an assigned timestamp. Currently, - these events can only be searched from the command line (when you don't specify a timestamp - filter). - ::: - -* `` are paths to JSON log files or directories containing such files. - * Each JSON log file should contain each log event as a [separate JSON object][json-log-format], - i.e., _not_ as an array. - -:::{tip} -To compress logs from object storage, see -[Using object storage](../guides/guides-using-object-storage/index). -::: - -# Sample logs - -For some sample logs, check out the open-source [datasets](../resources-datasets.md). - -# Examining compression statistics - -The compression script used above will output the compression ratio of each dataset you compress, or -you can use the UI to view overall statistics. - -[json-log-format]: ../quick-start-cluster-setup/index.md#clp-json diff --git a/docs/src/user-guide/quick-start-compression/text.md b/docs/src/user-guide/quick-start-compression/text.md deleted file mode 100644 index 29e798b9d8..0000000000 --- a/docs/src/user-guide/quick-start-compression/text.md +++ /dev/null @@ -1,18 +0,0 @@ -# Compressing unstructured text logs - -To compress unstructured text logs, from inside the package directory, run: - -```bash -sbin/compress.sh [ ...] -``` - -`` are paths to unstructured text log files or directories containing such files. - -# Sample logs - -For some sample logs, check out the open-source [datasets](../resources-datasets.md). - -# Examining compression statistics - -The compression script used above will output the compression ratio of each dataset you compress, or -you can use the UI to view overall statistics. diff --git a/docs/src/user-guide/quick-start-overview.md b/docs/src/user-guide/quick-start-overview.md deleted file mode 100644 index 9cf155a607..0000000000 --- a/docs/src/user-guide/quick-start-overview.md +++ /dev/null @@ -1,8 +0,0 @@ -# Overview - -CLP operates as a distributed system, so you'll need to set up a cluster before compressing and -searching your logs. Follow the guides below for each step: - -1. [Cluster setup](quick-start-cluster-setup/index) -2. [Compression](quick-start-compression/index) -3. [Search](quick-start-search/index) diff --git a/docs/src/user-guide/quick-start-search/cli-search.md b/docs/src/user-guide/quick-start-search/cli-search.md deleted file mode 100644 index b8d9061923..0000000000 --- a/docs/src/user-guide/quick-start-search/cli-search.md +++ /dev/null @@ -1,23 +0,0 @@ -# Searching from the command line - -From inside the package, run: - -``` -sbin/search.sh '' -``` - -The format of `` depends on the format your logs: [JSON](../reference-json-search-syntax.md) -or [unstructured text](../reference-text-search-syntax.md). - -To narrow your search to a specific time range: - -* Add `--begin-time ` to filter for log events after a certain time. - * `` is the timestamp as milliseconds since the UNIX epoch. -* Add `--end-time ` to filter for log events after a certain time. - -To perform case-insensitive searches, add the `--ignore-case` flag. - -:::{caution} -To match the convention of other tools, by default, searches are case-**insensitive** in the UI and -searches are case-**sensitive** on the command line. -::: diff --git a/docs/src/user-guide/quick-start-search/clp-search-ui.png b/docs/src/user-guide/quick-start-search/clp-search-ui.png deleted file mode 100644 index 74530978a8..0000000000 --- a/docs/src/user-guide/quick-start-search/clp-search-ui.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:80e2309f9b03856672edd73ae34dde67814ca04e2cd5235586b78a67fc4f18bf -size 57858 diff --git a/docs/src/user-guide/quick-start-search/index.md b/docs/src/user-guide/quick-start-search/index.md deleted file mode 100644 index 6e3ac9bc7b..0000000000 --- a/docs/src/user-guide/quick-start-search/index.md +++ /dev/null @@ -1,41 +0,0 @@ -# Search - -You can search your logs from the UI or from the command line. - -::::{grid} 1 1 2 2 -:gutter: 2 - -:::{grid-item-card} -:link: ui-search -Searching from the UI -::: - -:::{grid-item-card} -:link: cli-search -Searching from the command line -::: -:::: - -The syntax for searching JSON logs and unstructured text logs is different. - -::::{grid} 1 1 2 2 -:gutter: 2 - -:::{grid-item-card} -:link: ../reference-json-search-syntax -`clp-json` search syntax -::: - -:::{grid-item-card} -:link: ../reference-text-search-syntax -`clp-text` search syntax -::: -:::: - -:::{toctree} -:hidden: -:caption: Search - -ui-search -cli-search -::: diff --git a/docs/src/user-guide/quick-start-search/ui-search.md b/docs/src/user-guide/quick-start-search/ui-search.md deleted file mode 100644 index 32af5bb4b0..0000000000 --- a/docs/src/user-guide/quick-start-search/ui-search.md +++ /dev/null @@ -1,30 +0,0 @@ -# Searching from the UI - -CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default -(if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). - -:::{image} clp-search-ui.png -::: - -The image above shows the search page after running a query. The numbered circles correspond to the -following features: - -1. The search box is where you can enter a query. - * The format of a query depends on the format of your logs: - [JSON](../reference-json-search-syntax.md) or - [unstructured text](../reference-text-search-syntax.md). -2. The timeline shows the number of results across the time range of your query. - * You can click and drag to zoom into a time range or use the time range filter in (4). -3. The table displays the search results for your query. -4. Clicking the icon reveals additional filters for your query. - * The time range filter allows you to specify the period of time that matching log events must be - in. - * The case sensitivity filter allows you to specify whether CLP should respect the case of your - query. -5. Clicking the icon reveals options for displaying results. -6. The icon clears the results of the last query. - -:::{note} -By default, the UI will only return 1,000 of the latest search results. To perform searches which -return more results, use the [command line](cli-search.md). -::: \ No newline at end of file diff --git a/docs/src/user-guide/quick-start/clp-text/main-page.md b/docs/src/user-guide/quick-start/clp-text/main-page.md index a1e6efac36..0082c4f8d3 100644 --- a/docs/src/user-guide/quick-start/clp-text/main-page.md +++ b/docs/src/user-guide/quick-start/clp-text/main-page.md @@ -35,11 +35,11 @@ sbin/compress.sh [ ...] Compressed logs will be stored in the `/var/bin/archives` directory. -# Sample logs +### Sample logs For some sample logs, check out the open-source [datasets](../../resources-datasets.md). -# Examining compression statistics +### Examining compression statistics The compression script used above will output the compression ratio of each dataset you compress, or you can use the UI to view overall statistics. @@ -62,9 +62,6 @@ one * two?three would return all logs that contain the sequence of characters "one" followed by any number (even zero) of other characters, followed by "two", followed by one other character, followed by "three". - - - There are a number of other syntax rules specific to unstructured text queries that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference-text-search-syntax). ### Searching from the command line From f60aff26524c381d72bdb9de48e074de2df8cae7 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 5 Jun 2025 16:50:37 +0000 Subject: [PATCH 08/40] Restructured directories to better reflect website structure --- .../design-kv-ir-streams/background.md | 4 ++-- docs/src/index.md | 2 +- docs/src/user-guide/{ => core}/core-clp-s.md | 4 ++-- .../user-guide/{ => core}/core-container.md | 0 .../user-guide/{ => core}/core-overview.md | 0 .../{ => core}/core-unstructured/clp.md | 2 +- .../{ => core}/core-unstructured/glt.md | 0 .../{ => core}/core-unstructured/index.md | 0 .../make-dictionaries-readable.md | 0 docs/src/user-guide/index.md | 22 +++++++++---------- .../quick-start/clp-json/main-page.md | 4 ++-- .../quick-start/clp-text/main-page.md | 4 ++-- .../reference-json-search-syntax.md | 0 .../reference-text-search-syntax.md | 0 .../reference-unstructured-schema-file.md | 0 .../{ => resources}/resources-datasets.md | 0 16 files changed, 21 insertions(+), 21 deletions(-) rename docs/src/user-guide/{ => core}/core-clp-s.md (96%) rename docs/src/user-guide/{ => core}/core-container.md (100%) rename docs/src/user-guide/{ => core}/core-overview.md (100%) rename docs/src/user-guide/{ => core}/core-unstructured/clp.md (97%) rename docs/src/user-guide/{ => core}/core-unstructured/glt.md (100%) rename docs/src/user-guide/{ => core}/core-unstructured/index.md (100%) rename docs/src/user-guide/{ => core}/core-unstructured/make-dictionaries-readable.md (100%) rename docs/src/user-guide/{ => reference}/reference-json-search-syntax.md (100%) rename docs/src/user-guide/{ => reference}/reference-text-search-syntax.md (100%) rename docs/src/user-guide/{ => reference}/reference-unstructured-schema-file.md (100%) rename docs/src/user-guide/{ => resources}/resources-datasets.md (100%) diff --git a/docs/src/dev-guide/design-kv-ir-streams/background.md b/docs/src/dev-guide/design-kv-ir-streams/background.md index b7a86a21b3..d8f0f3f802 100644 --- a/docs/src/dev-guide/design-kv-ir-streams/background.md +++ b/docs/src/dev-guide/design-kv-ir-streams/background.md @@ -11,7 +11,7 @@ We discuss each below. ## clp-s compression -At a high-level, [clp-s](../../user-guide/core-clp-s.md) compresses log events into what we call +At a high-level, [clp-s](../../user-guide/core/core-clp-s.md) compresses log events into what we call archives. Depending on the configured size-threshold for each archive, a set of log events may be compressed into one or more archives. The archive format is designed so that each archive is self-contained and independent, allowing archives to be searched concurrently. @@ -318,7 +318,7 @@ the archive is complete. ## Parsing & encoding unstructured text -clp-s uses [clp](../../user-guide/core-unstructured/clp.md)'s algorithm to parse and encode +clp-s uses [clp](../../user-guide/core/core-unstructured/clp.md)'s algorithm to parse and encode unstructured text. Unstructured text is a string that contains zero or more variable values interspersed with non-variable (static) text. For example, in [Figure 1](#figure-1), log event #1's `message` value is unstructured text containing the variable values `task_1` and `2`. At a diff --git a/docs/src/index.md b/docs/src/index.md index 51fbc5f045..ae2405d1b9 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -109,7 +109,7 @@ dev-guide/index [clp-ffi-py]: https://github.com/y-scope/clp-ffi-py [clp-loglib-py]: https://github.com/y-scope/clp-loglib-py [clp-paper]: https://www.usenix.org/system/files/osdi21-rodrigues.pdf -[datasets]: user-guide/resources-datasets +[datasets]: user-guide/resources/resources-datasets [feature-req]: https://github.com/y-scope/clp/issues/new?assignees=&labels=enhancement&template=feature-request.yml [log-surgeon]: https://github.com/y-scope/log-surgeon [log-viewer]: https://github.com/y-scope/yscope-log-viewer diff --git a/docs/src/user-guide/core-clp-s.md b/docs/src/user-guide/core/core-clp-s.md similarity index 96% rename from docs/src/user-guide/core-clp-s.md rename to docs/src/user-guide/core/core-clp-s.md index 8528f98d2f..614166d394 100644 --- a/docs/src/user-guide/core-clp-s.md +++ b/docs/src/user-guide/core/core-clp-s.md @@ -78,7 +78,7 @@ Usage: ``` * `archives-dir` is a directory containing archives. -* `kql-query` is a [KQL](reference-json-search-syntax) query. +* `kql-query` is a [KQL](../reference/reference-json-search-syntax) query. * `options` allow you to specify things like a specific archive (from within `archives-dir`) to search (`--archive-id `). * For a complete list, run `./clp-s s --help` @@ -124,4 +124,4 @@ compressed data:** * The input directory structure is not preserved and during decompression all files are written to the same file. * In addition, there are a few limitations, related to querying arrays, described in the search - syntax [reference](reference-json-search-syntax). + syntax [reference](../reference/reference-json-search-syntax). diff --git a/docs/src/user-guide/core-container.md b/docs/src/user-guide/core/core-container.md similarity index 100% rename from docs/src/user-guide/core-container.md rename to docs/src/user-guide/core/core-container.md diff --git a/docs/src/user-guide/core-overview.md b/docs/src/user-guide/core/core-overview.md similarity index 100% rename from docs/src/user-guide/core-overview.md rename to docs/src/user-guide/core/core-overview.md diff --git a/docs/src/user-guide/core-unstructured/clp.md b/docs/src/user-guide/core/core-unstructured/clp.md similarity index 97% rename from docs/src/user-guide/core-unstructured/clp.md rename to docs/src/user-guide/core/core-unstructured/clp.md index ddcace4d70..02ae210823 100644 --- a/docs/src/user-guide/core-unstructured/clp.md +++ b/docs/src/user-guide/core/core-unstructured/clp.md @@ -17,7 +17,7 @@ Usage: * You can use the same directory repeatedly and `clp` will add to the compressed logs within. * `input-path` is any plain-text log file or directory containing such files. * `options` allow you to specify things like a path to a custom - [schema](../reference-unstructured-schema-file) file (`--schema-path `). + [schema](../../reference/reference-unstructured-schema-file) file (`--schema-path `). * For a complete list, run `./clp c --help` ### Examples diff --git a/docs/src/user-guide/core-unstructured/glt.md b/docs/src/user-guide/core/core-unstructured/glt.md similarity index 100% rename from docs/src/user-guide/core-unstructured/glt.md rename to docs/src/user-guide/core/core-unstructured/glt.md diff --git a/docs/src/user-guide/core-unstructured/index.md b/docs/src/user-guide/core/core-unstructured/index.md similarity index 100% rename from docs/src/user-guide/core-unstructured/index.md rename to docs/src/user-guide/core/core-unstructured/index.md diff --git a/docs/src/user-guide/core-unstructured/make-dictionaries-readable.md b/docs/src/user-guide/core/core-unstructured/make-dictionaries-readable.md similarity index 100% rename from docs/src/user-guide/core-unstructured/make-dictionaries-readable.md rename to docs/src/user-guide/core/core-unstructured/make-dictionaries-readable.md diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index 59bb85bb65..ee4cedb6da 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -23,21 +23,21 @@ Guides for using CLP in a variety of use cases. ::: :::{grid-item-card} -:link: core-overview +:link: core/core-overview Core ^^^ Docs about CLP's core component for compressing, decompressing, and searching logs. ::: :::{grid-item-card} -:link: resources-datasets +:link: resources/resources-datasets Resources ^^^ Resources like log datasets, etc. ::: :::{grid-item-card} -:link: reference-json-search-syntax +:link: reference/reference-json-search-syntax Reference ^^^ Reference docs like format specifications, etc. @@ -71,24 +71,24 @@ guides/guides-multi-node/multi-node :caption: Core :glob: -core-overview -core-container -core-clp-s -core-unstructured/index +core/core-overview +core/core-container +core/core-clp-s +core/core-unstructured/index ::: :::{toctree} :hidden: :caption: Resources -resources-datasets +resources/resources-datasets ::: :::{toctree} :hidden: :caption: Reference -reference-json-search-syntax -reference-text-search-syntax -reference-unstructured-schema-file +reference/reference-json-search-syntax +reference/reference-text-search-syntax +reference/reference-unstructured-schema-file ::: diff --git a/docs/src/user-guide/quick-start/clp-json/main-page.md b/docs/src/user-guide/quick-start/clp-json/main-page.md index ea596f09da..b8cbc3f071 100644 --- a/docs/src/user-guide/quick-start/clp-json/main-page.md +++ b/docs/src/user-guide/quick-start/clp-json/main-page.md @@ -55,7 +55,7 @@ To compress logs from object storage, see ### Sample logs -For some sample logs, check out the open-source [datasets](../../resources-datasets.md). +For some sample logs, check out the open-source [datasets](../../resources/resources-datasets.md). ### Examining compression statistics @@ -82,7 +82,7 @@ where `key` is the name of the JSON key you'd like to search within, and `value` key1: value1 AND key2: value2 OR key3: value3 ... ``` -There are a number of other JSON-specific syntax rules that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference-json-search-syntax). +There are a number of other JSON-specific syntax rules that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference/reference-json-search-syntax). ### Searching from the command line diff --git a/docs/src/user-guide/quick-start/clp-text/main-page.md b/docs/src/user-guide/quick-start/clp-text/main-page.md index 0082c4f8d3..de8324fa8a 100644 --- a/docs/src/user-guide/quick-start/clp-text/main-page.md +++ b/docs/src/user-guide/quick-start/clp-text/main-page.md @@ -37,7 +37,7 @@ Compressed logs will be stored in the `/var/bin/archives` directory. ### Sample logs -For some sample logs, check out the open-source [datasets](../../resources-datasets.md). +For some sample logs, check out the open-source [datasets](../../resources/resources-datasets.md). ### Examining compression statistics @@ -62,7 +62,7 @@ one * two?three would return all logs that contain the sequence of characters "one" followed by any number (even zero) of other characters, followed by "two", followed by one other character, followed by "three". -There are a number of other syntax rules specific to unstructured text queries that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference-text-search-syntax). +There are a number of other syntax rules specific to unstructured text queries that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference/reference-text-search-syntax). ### Searching from the command line diff --git a/docs/src/user-guide/reference-json-search-syntax.md b/docs/src/user-guide/reference/reference-json-search-syntax.md similarity index 100% rename from docs/src/user-guide/reference-json-search-syntax.md rename to docs/src/user-guide/reference/reference-json-search-syntax.md diff --git a/docs/src/user-guide/reference-text-search-syntax.md b/docs/src/user-guide/reference/reference-text-search-syntax.md similarity index 100% rename from docs/src/user-guide/reference-text-search-syntax.md rename to docs/src/user-guide/reference/reference-text-search-syntax.md diff --git a/docs/src/user-guide/reference-unstructured-schema-file.md b/docs/src/user-guide/reference/reference-unstructured-schema-file.md similarity index 100% rename from docs/src/user-guide/reference-unstructured-schema-file.md rename to docs/src/user-guide/reference/reference-unstructured-schema-file.md diff --git a/docs/src/user-guide/resources-datasets.md b/docs/src/user-guide/resources/resources-datasets.md similarity index 100% rename from docs/src/user-guide/resources-datasets.md rename to docs/src/user-guide/resources/resources-datasets.md From 3e29ed16cc69f37a331ddf598da32c1a55808ba5 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 5 Jun 2025 17:18:22 +0000 Subject: [PATCH 09/40] Add landing page for 'Reference' --- docs/src/user-guide/index.md | 3 ++- docs/src/user-guide/reference/index.md | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 docs/src/user-guide/reference/index.md diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index ee4cedb6da..22fc130661 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -37,7 +37,7 @@ Resources like log datasets, etc. ::: :::{grid-item-card} -:link: reference/reference-json-search-syntax +:link: reference/index Reference ^^^ Reference docs like format specifications, etc. @@ -88,6 +88,7 @@ resources/resources-datasets :hidden: :caption: Reference +reference/index reference/reference-json-search-syntax reference/reference-text-search-syntax reference/reference-unstructured-schema-file diff --git a/docs/src/user-guide/reference/index.md b/docs/src/user-guide/reference/index.md new file mode 100644 index 0000000000..5341702ca7 --- /dev/null +++ b/docs/src/user-guide/reference/index.md @@ -0,0 +1,23 @@ +# Overview + +This section compiles information on syntax rules and format specifications relevant to CLP. + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: reference-json-search-syntax +clp-json search syntax +::: + +:::{grid-item-card} +:link: reference-text-search-syntax +clp-text search syntax +::: + +:::{grid-item-card} +:link: reference-unstructured-schema-file +Schema file syntax +::: + +:::: From 596c70caa781f4ec2f3b9da5a770a228179e113c Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 5 Jun 2025 17:33:03 +0000 Subject: [PATCH 10/40] Add link at end of clp-json and clp-text quickstart pages to their respective core pages --- docs/src/user-guide/quick-start/clp-json/main-page.md | 6 ++++++ docs/src/user-guide/quick-start/clp-text/main-page.md | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/docs/src/user-guide/quick-start/clp-json/main-page.md b/docs/src/user-guide/quick-start/clp-json/main-page.md index b8cbc3f071..1f2ee6cf1d 100644 --- a/docs/src/user-guide/quick-start/clp-json/main-page.md +++ b/docs/src/user-guide/quick-start/clp-json/main-page.md @@ -140,3 +140,9 @@ If you need to stop CLP, run the command ```bash sbin/stop-clp.sh ``` + +--- + +## More information + +You've reached the end of the clp-json quick start guide. For more information on clp-json, visit the [CLP for JSON logs](../../core/core-clp-s) page. diff --git a/docs/src/user-guide/quick-start/clp-text/main-page.md b/docs/src/user-guide/quick-start/clp-text/main-page.md index de8324fa8a..75b5abb9ef 100644 --- a/docs/src/user-guide/quick-start/clp-text/main-page.md +++ b/docs/src/user-guide/quick-start/clp-text/main-page.md @@ -120,3 +120,9 @@ If you need to stop CLP, run the command ```bash sbin/stop-clp.sh ``` + +--- + +## More information + +You've reached the end of the clp-text quick start guide. For more information on clp-text, visit the [CLP for unstructured logs](../../core/core-unstructured/index) page. From 2692e50022d3290b465454d20262c38ff86ef134 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 5 Jun 2025 18:29:59 +0000 Subject: [PATCH 11/40] general edits (and make rabbit happy) --- .../guides/guides-multi-node/multi-node.md | 4 ++-- .../guides-using-object-storage/clp-usage.md | 2 +- docs/src/user-guide/index.md | 8 ++++---- .../quick-start/clp-json/main-page.md | 8 ++++---- .../quick-start/clp-text/main-page.md | 18 +++++++++--------- docs/src/user-guide/quick-start/index.md | 2 +- .../user-guide/quick-start/setup/main-page.md | 15 +++++++-------- 7 files changed, 28 insertions(+), 29 deletions(-) diff --git a/docs/src/user-guide/guides/guides-multi-node/multi-node.md b/docs/src/user-guide/guides/guides-multi-node/multi-node.md index 461b7b49ee..11250c95dc 100644 --- a/docs/src/user-guide/guides/guides-multi-node/multi-node.md +++ b/docs/src/user-guide/guides/guides-multi-node/multi-node.md @@ -117,14 +117,14 @@ Check out the quick-start guide for your chosen flavour of CLP to learn how to c :::{grid-item-card} :link: ../../../quick-start/clp-json/main-page -CLP-JSON +clp-json ^^^ Compress and search JSON logs. ::: :::{grid-item-card} :link: ../../../quick-start/clp-text/main-page -CLP-text +clp-text ^^^ Compress and search unstructured text logs. ::: diff --git a/docs/src/user-guide/guides/guides-using-object-storage/clp-usage.md b/docs/src/user-guide/guides/guides-using-object-storage/clp-usage.md index 54f6aa7fdb..15240cf7fe 100644 --- a/docs/src/user-guide/guides/guides-using-object-storage/clp-usage.md +++ b/docs/src/user-guide/guides/guides-using-object-storage/clp-usage.md @@ -1,7 +1,7 @@ # Using CLP with object storage To compress logs from S3, follow the steps in the section below. For all other operations, you -should be able to use CLP as described in the [clp-json quick start guide](../../quick-start/clp-json/main-page). +should be able to use CLP as described in the [clp-json quick-start guide](../../quick-start/clp-json/main-page). ## Compressing logs from S3 diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index 22fc130661..892c64469e 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -10,16 +10,16 @@ The sections are as follows: :::{grid-item-card} :link: quick-start/index -Quick start +Quick-start ^^^ -A quick start guide for choosing a flavour of CLP, setting it up, compressing your logs, and searching them. +A quick-start guide for choosing a flavour of CLP, setting it up, compressing your logs, and searching them. ::: :::{grid-item-card} :link: guides/guides-overview Guides ^^^ -Guides for using CLP in a variety of use cases. +Guides for using CLP in various use cases. ::: :::{grid-item-card} @@ -46,7 +46,7 @@ Reference docs like format specifications, etc. :::{toctree} :hidden: -:caption: Quick start +:caption: quick-start quick-start/index quick-start/setup/main-page diff --git a/docs/src/user-guide/quick-start/clp-json/main-page.md b/docs/src/user-guide/quick-start/clp-json/main-page.md index 1f2ee6cf1d..d2eb4d3ea7 100644 --- a/docs/src/user-guide/quick-start/clp-json/main-page.md +++ b/docs/src/user-guide/quick-start/clp-json/main-page.md @@ -1,6 +1,6 @@ -# CLP-JSON +# clp-json Quick-start -This page will walk you through how to start up CLP, and also show you how to use CLP to compress and search JSON logs. +This page will walk you through how to start up CLP, and then use it to compress and search JSON logs. :::{caution} If you're using the `clp-json` release, you can only compress JSON logs. @@ -55,7 +55,7 @@ To compress logs from object storage, see ### Sample logs -For some sample logs, check out the open-source [datasets](../../resources/resources-datasets.md). +For some sample logs, check out the open-source [datasets](../../resources/resources-datasets). ### Examining compression statistics @@ -145,4 +145,4 @@ sbin/stop-clp.sh ## More information -You've reached the end of the clp-json quick start guide. For more information on clp-json, visit the [CLP for JSON logs](../../core/core-clp-s) page. +You've reached the end of the clp-json quick-start guide. For more information on clp-json, visit the [CLP for JSON logs](../../core/core-clp-s) page. diff --git a/docs/src/user-guide/quick-start/clp-text/main-page.md b/docs/src/user-guide/quick-start/clp-text/main-page.md index 75b5abb9ef..05f1ed3a90 100644 --- a/docs/src/user-guide/quick-start/clp-text/main-page.md +++ b/docs/src/user-guide/quick-start/clp-text/main-page.md @@ -1,9 +1,9 @@ -# CLP-text +# clp-text Quick-start -This page will walk you through how to start up CLP, and also show you how to use CLP to compress and search unstructured text logs. +This page will walk you through how to start up CLP, and then use it to compress and search unstructured text logs. :::{caution} -If you're using the `clp-text` release, you should only compress unstructured text logs. `clp-text` can compress and search JSON logs as if it was unstructured text, but `clp-text` cannot query individual fields. This limitation will be addressed in a future version of CLP. +If you're using the `clp-text` release, you should only compress unstructured text logs. `clp-text` is able to compress and search JSON logs as if it was unstructured text, but `clp-text` cannot query individual fields. This limitation will be addressed in a future version of CLP. ::: --- @@ -37,7 +37,7 @@ Compressed logs will be stored in the `/var/bin/archives` directory. ### Sample logs -For some sample logs, check out the open-source [datasets](../../resources/resources-datasets.md). +For some sample logs, check out the open-source [datasets](../../resources/resources-datasets). ### Examining compression statistics @@ -54,13 +54,13 @@ You can search through your logs using queries from the UI or from the command l Regardless of what method you use to search, you'll need a query to find the logs you're looking for. All unstructured text log queries are written as plain text. -You can use a couple special characters to make these queries more versatile. `*` can be used as a placeholder for an unknown number of characters, and `?` can be used for a single character. For example, the query +You can use a couple of special characters to make these queries more versatile. `*` can be used as a placeholder for an unknown number of characters, and `?` can be used for a single character. For example, the query -``` bash -one * two?three +```bash +a*b?c ``` -would return all logs that contain the sequence of characters "one" followed by any number (even zero) of other characters, followed by "two", followed by one other character, followed by "three". +would return all logs that contain the character `"a"` followed by any number (including zero) of other characters, followed by `"b"`, followed by one other character, followed by `"c"`. There are a number of other syntax rules specific to unstructured text queries that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference/reference-text-search-syntax). @@ -125,4 +125,4 @@ sbin/stop-clp.sh ## More information -You've reached the end of the clp-text quick start guide. For more information on clp-text, visit the [CLP for unstructured logs](../../core/core-unstructured/index) page. +You've reached the end of the clp-text quick-start guide. For more information on clp-text, visit the [CLP for unstructured logs](../../core/core-unstructured/index) page. diff --git a/docs/src/user-guide/quick-start/index.md b/docs/src/user-guide/quick-start/index.md index be4a7cccb5..79e140db23 100644 --- a/docs/src/user-guide/quick-start/index.md +++ b/docs/src/user-guide/quick-start/index.md @@ -1,6 +1,6 @@ # Overview -To learn how to use CLP, follow the "Setup" link. +If you're new to CLP, follow the link below to learn how to set it up. ::::{grid} 1 1 1 1 :gutter: 2 diff --git a/docs/src/user-guide/quick-start/setup/main-page.md b/docs/src/user-guide/quick-start/setup/main-page.md index bef5edc331..3cfc327ea9 100644 --- a/docs/src/user-guide/quick-start/setup/main-page.md +++ b/docs/src/user-guide/quick-start/setup/main-page.md @@ -12,7 +12,7 @@ To run properly on your system, CLP requires a few other programs. Before you se CLP uses Docker to deploy its different components. You can check whether Docker is already installed on your system by running the command -``` bash +```bash docker version ``` @@ -26,11 +26,11 @@ If you're using Docker Desktop, ensure version 4.34 or higher is installed, and CLP uses Python to interpret the scripts that coordinate how it runs. Specifically, CLP needs Python version 3.8 or higher. Many Linux distributions come with Python pre-installed, but just to be sure, run the command -``` bash +```bash python3 --version ``` -If Python isn't on your system, or if the version isn't high enough, follow the instructions [here][Python] to install or upgrade it. +If Python isn't on your system, or if the version isn't high enough, install or upgrade it. :::{note} If you're planning on deploying CLP on multiple nodes/systems, there are a few other system requirements; check out the [multi-node deployment](../../guides/guides-multi-node/multi-node) page for more details. @@ -74,7 +74,7 @@ object. For example: The log file above contains two log events represented by two JSON objects printed one after the other. Whitespace is ignored, so the log events could also appear with no newlines and indentation. -If your logs are JSON logs, download and extract the JSON flavour tarball from the [Releases][clp-releases] page on GitHub, and proceed to the CLP-JSON portal below. +If your logs are JSON logs, download and extract the JSON flavour tarball from the [Releases][clp-releases] page on GitHub, and proceed to the clp-json portal below. ::::{grid} 1 1 1 1 :gutter: 2 @@ -83,7 +83,7 @@ If your logs are JSON logs, download and extract the JSON flavour tarball from t :link: ../clp-json/main-page CLP for JSON logs ^^^ -Learn how to start up CLP-JSON, and begin compressing and searching JSON logs. +Learn how to start up clp-json, and begin compressing and searching JSON logs. ::: :::: @@ -112,7 +112,7 @@ Caused by: java.net.UnknownHostException: i-e5d112ea The log file above contains two log events, both beginning with a timestamp. The first is a single line while the second contains multiple lines. -If your logs are unstructured text logs, download and extract the unstructured text flavour tarball from the [Releases][clp-releases] page on GitHub, and proceed to the CLP-text portal below. +If your logs are unstructured text logs, download and extract the unstructured text flavour tarball from the [Releases][clp-releases] page on GitHub, and proceed to the clp-text portal below. ::::{grid} 1 1 1 1 :gutter: 2 @@ -121,12 +121,11 @@ If your logs are unstructured text logs, download and extract the unstructured t :link: ../clp-text/main-page CLP for unstructured logs ^^^ -Learn how to start up CLP-text, and begin compressing and searching unstructured text logs. +Learn how to start up clp-text, and begin compressing and searching unstructured text logs. ::: :::: [Docker]: https://docs.docker.com/engine/install/ [docker-non-root]: https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user [docker-desktop-host-networking]: https://docs.docker.com/engine/network/drivers/host/#docker-desktop -[Python]: https://www.geeksforgeeks.org/how-to-install-python-on-linux/ [clp-releases]: https://github.com/y-scope/clp/releases From 62d38ea2810a046ffc782d5c84e0b4da3e92bfc3 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 5 Jun 2025 18:46:01 +0000 Subject: [PATCH 12/40] more rabbit edits --- .../guides/guides-multi-node/multi-node.md | 2 +- docs/src/user-guide/quick-start/setup/main-page.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/src/user-guide/guides/guides-multi-node/multi-node.md b/docs/src/user-guide/guides/guides-multi-node/multi-node.md index 11250c95dc..7444abecf0 100644 --- a/docs/src/user-guide/guides/guides-multi-node/multi-node.md +++ b/docs/src/user-guide/guides/guides-multi-node/multi-node.md @@ -110,7 +110,7 @@ Where `` is the name of the component in the groups above. ## Using CLP -Check out the quick-start guide for your chosen flavour of CLP to learn how to compress and search your logs. +Check out the [quick-start guide](../../quick-start/index) for your chosen flavour of CLP to learn how to compress and search your logs. ::::{grid} 1 1 2 2 :gutter: 2 diff --git a/docs/src/user-guide/quick-start/setup/main-page.md b/docs/src/user-guide/quick-start/setup/main-page.md index 3cfc327ea9..e132642d78 100644 --- a/docs/src/user-guide/quick-start/setup/main-page.md +++ b/docs/src/user-guide/quick-start/setup/main-page.md @@ -16,7 +16,7 @@ CLP uses Docker to deploy its different components. You can check whether Docker docker version ``` -If Docker is not on your system, follow the instructions [here][Docker] to install it. +If Docker is not on your system, follow the instructions [here][Docker] to install it. If you're not running as root, ensure Docker can be run [without superuser privileges][docker-non-root]. @@ -24,7 +24,7 @@ If you're using Docker Desktop, ensure version 4.34 or higher is installed, and ### Python -CLP uses Python to interpret the scripts that coordinate how it runs. Specifically, CLP needs Python version 3.8 or higher. Many Linux distributions come with Python pre-installed, but just to be sure, run the command +CLP uses Python to interpret the scripts that coordinate how it runs. Specifically, CLP needs Python version 3.8 or higher. Many Linux distributions come with Python pre-installed; to confirm that it's on your system, run the command ```bash python3 --version @@ -33,7 +33,7 @@ python3 --version If Python isn't on your system, or if the version isn't high enough, install or upgrade it. :::{note} -If you're planning on deploying CLP on multiple nodes/systems, there are a few other system requirements; check out the [multi-node deployment](../../guides/guides-multi-node/multi-node) page for more details. +If you're planning to deploy CLP on multiple nodes/systems, there are a few other system requirements; check out the [multi-node deployment](../../guides/guides-multi-node/multi-node) page for more details. ::: --- @@ -52,7 +52,7 @@ Both flavours contain the same binaries but are configured with different values ### clp-json -The JSON flavour of CLP is appropriate for JSON logs where each log event is an independent JSON +The JSON flavour of CLP is appropriate for JSON logs, where each log event is an independent JSON object. For example: ```json lines @@ -89,7 +89,7 @@ Learn how to start up clp-json, and begin compressing and searching JSON logs. ### clp-text -The unstructured text flavour of CLP is appropriate for unstructured text logs where each log event contains a +The unstructured text flavour of CLP is appropriate for unstructured text logs, where each log event contains a timestamp and may span one or more lines. :::{note} @@ -99,7 +99,7 @@ it will treat each line as an independent log event. For example: -``` +```text 2015-03-23T15:50:17.926Z INFO container_1 Transitioned from ALLOCATED to ACQUIRED 2015-03-23T15:50:17.927Z ERROR Scheduler: Error trying to assign container token java.lang.IllegalArgumentException: java.net.UnknownHostException: i-e5d112ea From ab525d28109da396a363ae3043d9fdd51042abe4 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 5 Jun 2025 19:16:26 +0000 Subject: [PATCH 13/40] Fix broken link --- docs/src/user-guide/guides/guides-multi-node/multi-node.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user-guide/guides/guides-multi-node/multi-node.md b/docs/src/user-guide/guides/guides-multi-node/multi-node.md index 7444abecf0..ea6c461601 100644 --- a/docs/src/user-guide/guides/guides-multi-node/multi-node.md +++ b/docs/src/user-guide/guides/guides-multi-node/multi-node.md @@ -116,14 +116,14 @@ Check out the [quick-start guide](../../quick-start/index) for your chosen flavo :gutter: 2 :::{grid-item-card} -:link: ../../../quick-start/clp-json/main-page +:link: ../../quick-start/clp-json/main-page clp-json ^^^ Compress and search JSON logs. ::: :::{grid-item-card} -:link: ../../../quick-start/clp-text/main-page +:link: ../../quick-start/clp-text/main-page clp-text ^^^ Compress and search unstructured text logs. From 23919ccd057333f4bb2c377eb4dc5f1786b46f5c Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 5 Jun 2025 19:34:38 +0000 Subject: [PATCH 14/40] Reverted changes unrelated to the separation of clp-json and clp-text --- .../design-kv-ir-streams/background.md | 4 +-- docs/src/index.md | 2 +- docs/src/user-guide/{core => }/core-clp-s.md | 4 +-- .../user-guide/{core => }/core-container.md | 0 .../user-guide/{core => }/core-overview.md | 0 .../{core => }/core-unstructured/clp.md | 2 +- .../{core => }/core-unstructured/glt.md | 0 .../{core => }/core-unstructured/index.md | 0 .../make-dictionaries-readable.md | 0 docs/src/user-guide/index.md | 25 +++++++++---------- .../quick-start/clp-json/main-page.md | 6 ++--- .../quick-start/clp-text/main-page.md | 6 ++--- .../reference-json-search-syntax.md | 0 .../reference-text-search-syntax.md | 0 .../reference-unstructured-schema-file.md | 0 docs/src/user-guide/reference/index.md | 23 ----------------- .../{resources => }/resources-datasets.md | 0 17 files changed, 24 insertions(+), 48 deletions(-) rename docs/src/user-guide/{core => }/core-clp-s.md (96%) rename docs/src/user-guide/{core => }/core-container.md (100%) rename docs/src/user-guide/{core => }/core-overview.md (100%) rename docs/src/user-guide/{core => }/core-unstructured/clp.md (97%) rename docs/src/user-guide/{core => }/core-unstructured/glt.md (100%) rename docs/src/user-guide/{core => }/core-unstructured/index.md (100%) rename docs/src/user-guide/{core => }/core-unstructured/make-dictionaries-readable.md (100%) rename docs/src/user-guide/{reference => }/reference-json-search-syntax.md (100%) rename docs/src/user-guide/{reference => }/reference-text-search-syntax.md (100%) rename docs/src/user-guide/{reference => }/reference-unstructured-schema-file.md (100%) delete mode 100644 docs/src/user-guide/reference/index.md rename docs/src/user-guide/{resources => }/resources-datasets.md (100%) diff --git a/docs/src/dev-guide/design-kv-ir-streams/background.md b/docs/src/dev-guide/design-kv-ir-streams/background.md index d8f0f3f802..b7a86a21b3 100644 --- a/docs/src/dev-guide/design-kv-ir-streams/background.md +++ b/docs/src/dev-guide/design-kv-ir-streams/background.md @@ -11,7 +11,7 @@ We discuss each below. ## clp-s compression -At a high-level, [clp-s](../../user-guide/core/core-clp-s.md) compresses log events into what we call +At a high-level, [clp-s](../../user-guide/core-clp-s.md) compresses log events into what we call archives. Depending on the configured size-threshold for each archive, a set of log events may be compressed into one or more archives. The archive format is designed so that each archive is self-contained and independent, allowing archives to be searched concurrently. @@ -318,7 +318,7 @@ the archive is complete. ## Parsing & encoding unstructured text -clp-s uses [clp](../../user-guide/core/core-unstructured/clp.md)'s algorithm to parse and encode +clp-s uses [clp](../../user-guide/core-unstructured/clp.md)'s algorithm to parse and encode unstructured text. Unstructured text is a string that contains zero or more variable values interspersed with non-variable (static) text. For example, in [Figure 1](#figure-1), log event #1's `message` value is unstructured text containing the variable values `task_1` and `2`. At a diff --git a/docs/src/index.md b/docs/src/index.md index ae2405d1b9..51fbc5f045 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -109,7 +109,7 @@ dev-guide/index [clp-ffi-py]: https://github.com/y-scope/clp-ffi-py [clp-loglib-py]: https://github.com/y-scope/clp-loglib-py [clp-paper]: https://www.usenix.org/system/files/osdi21-rodrigues.pdf -[datasets]: user-guide/resources/resources-datasets +[datasets]: user-guide/resources-datasets [feature-req]: https://github.com/y-scope/clp/issues/new?assignees=&labels=enhancement&template=feature-request.yml [log-surgeon]: https://github.com/y-scope/log-surgeon [log-viewer]: https://github.com/y-scope/yscope-log-viewer diff --git a/docs/src/user-guide/core/core-clp-s.md b/docs/src/user-guide/core-clp-s.md similarity index 96% rename from docs/src/user-guide/core/core-clp-s.md rename to docs/src/user-guide/core-clp-s.md index 614166d394..8528f98d2f 100644 --- a/docs/src/user-guide/core/core-clp-s.md +++ b/docs/src/user-guide/core-clp-s.md @@ -78,7 +78,7 @@ Usage: ``` * `archives-dir` is a directory containing archives. -* `kql-query` is a [KQL](../reference/reference-json-search-syntax) query. +* `kql-query` is a [KQL](reference-json-search-syntax) query. * `options` allow you to specify things like a specific archive (from within `archives-dir`) to search (`--archive-id `). * For a complete list, run `./clp-s s --help` @@ -124,4 +124,4 @@ compressed data:** * The input directory structure is not preserved and during decompression all files are written to the same file. * In addition, there are a few limitations, related to querying arrays, described in the search - syntax [reference](../reference/reference-json-search-syntax). + syntax [reference](reference-json-search-syntax). diff --git a/docs/src/user-guide/core/core-container.md b/docs/src/user-guide/core-container.md similarity index 100% rename from docs/src/user-guide/core/core-container.md rename to docs/src/user-guide/core-container.md diff --git a/docs/src/user-guide/core/core-overview.md b/docs/src/user-guide/core-overview.md similarity index 100% rename from docs/src/user-guide/core/core-overview.md rename to docs/src/user-guide/core-overview.md diff --git a/docs/src/user-guide/core/core-unstructured/clp.md b/docs/src/user-guide/core-unstructured/clp.md similarity index 97% rename from docs/src/user-guide/core/core-unstructured/clp.md rename to docs/src/user-guide/core-unstructured/clp.md index 02ae210823..ddcace4d70 100644 --- a/docs/src/user-guide/core/core-unstructured/clp.md +++ b/docs/src/user-guide/core-unstructured/clp.md @@ -17,7 +17,7 @@ Usage: * You can use the same directory repeatedly and `clp` will add to the compressed logs within. * `input-path` is any plain-text log file or directory containing such files. * `options` allow you to specify things like a path to a custom - [schema](../../reference/reference-unstructured-schema-file) file (`--schema-path `). + [schema](../reference-unstructured-schema-file) file (`--schema-path `). * For a complete list, run `./clp c --help` ### Examples diff --git a/docs/src/user-guide/core/core-unstructured/glt.md b/docs/src/user-guide/core-unstructured/glt.md similarity index 100% rename from docs/src/user-guide/core/core-unstructured/glt.md rename to docs/src/user-guide/core-unstructured/glt.md diff --git a/docs/src/user-guide/core/core-unstructured/index.md b/docs/src/user-guide/core-unstructured/index.md similarity index 100% rename from docs/src/user-guide/core/core-unstructured/index.md rename to docs/src/user-guide/core-unstructured/index.md diff --git a/docs/src/user-guide/core/core-unstructured/make-dictionaries-readable.md b/docs/src/user-guide/core-unstructured/make-dictionaries-readable.md similarity index 100% rename from docs/src/user-guide/core/core-unstructured/make-dictionaries-readable.md rename to docs/src/user-guide/core-unstructured/make-dictionaries-readable.md diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index 892c64469e..ffc250ac41 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -23,21 +23,21 @@ Guides for using CLP in various use cases. ::: :::{grid-item-card} -:link: core/core-overview +:link: core-overview Core ^^^ Docs about CLP's core component for compressing, decompressing, and searching logs. ::: :::{grid-item-card} -:link: resources/resources-datasets +:link: resources-datasets Resources ^^^ Resources like log datasets, etc. ::: :::{grid-item-card} -:link: reference/index +:link: reference-json-search-syntax Reference ^^^ Reference docs like format specifications, etc. @@ -46,7 +46,7 @@ Reference docs like format specifications, etc. :::{toctree} :hidden: -:caption: quick-start +:caption: Quick-start quick-start/index quick-start/setup/main-page @@ -71,25 +71,24 @@ guides/guides-multi-node/multi-node :caption: Core :glob: -core/core-overview -core/core-container -core/core-clp-s -core/core-unstructured/index +core-overview +core-container +core-clp-s +core-unstructured/index ::: :::{toctree} :hidden: :caption: Resources -resources/resources-datasets +resources-datasets ::: :::{toctree} :hidden: :caption: Reference -reference/index -reference/reference-json-search-syntax -reference/reference-text-search-syntax -reference/reference-unstructured-schema-file +reference-json-search-syntax +reference-text-search-syntax +reference-unstructured-schema-file ::: diff --git a/docs/src/user-guide/quick-start/clp-json/main-page.md b/docs/src/user-guide/quick-start/clp-json/main-page.md index d2eb4d3ea7..a08b965b96 100644 --- a/docs/src/user-guide/quick-start/clp-json/main-page.md +++ b/docs/src/user-guide/quick-start/clp-json/main-page.md @@ -55,7 +55,7 @@ To compress logs from object storage, see ### Sample logs -For some sample logs, check out the open-source [datasets](../../resources/resources-datasets). +For some sample logs, check out the open-source [datasets](../../resources-datasets). ### Examining compression statistics @@ -82,7 +82,7 @@ where `key` is the name of the JSON key you'd like to search within, and `value` key1: value1 AND key2: value2 OR key3: value3 ... ``` -There are a number of other JSON-specific syntax rules that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference/reference-json-search-syntax). +There are a number of other JSON-specific syntax rules that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference-json-search-syntax). ### Searching from the command line @@ -145,4 +145,4 @@ sbin/stop-clp.sh ## More information -You've reached the end of the clp-json quick-start guide. For more information on clp-json, visit the [CLP for JSON logs](../../core/core-clp-s) page. +You've reached the end of the clp-json quick-start guide. For more information on clp-json, visit the [CLP for JSON logs](../../core-clp-s) page. diff --git a/docs/src/user-guide/quick-start/clp-text/main-page.md b/docs/src/user-guide/quick-start/clp-text/main-page.md index 05f1ed3a90..a378ed9b90 100644 --- a/docs/src/user-guide/quick-start/clp-text/main-page.md +++ b/docs/src/user-guide/quick-start/clp-text/main-page.md @@ -37,7 +37,7 @@ Compressed logs will be stored in the `/var/bin/archives` directory. ### Sample logs -For some sample logs, check out the open-source [datasets](../../resources/resources-datasets). +For some sample logs, check out the open-source [datasets](../../resources-datasets). ### Examining compression statistics @@ -62,7 +62,7 @@ a*b?c would return all logs that contain the character `"a"` followed by any number (including zero) of other characters, followed by `"b"`, followed by one other character, followed by `"c"`. -There are a number of other syntax rules specific to unstructured text queries that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference/reference-text-search-syntax). +There are a number of other syntax rules specific to unstructured text queries that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference-text-search-syntax). ### Searching from the command line @@ -125,4 +125,4 @@ sbin/stop-clp.sh ## More information -You've reached the end of the clp-text quick-start guide. For more information on clp-text, visit the [CLP for unstructured logs](../../core/core-unstructured/index) page. +You've reached the end of the clp-text quick-start guide. For more information on clp-text, visit the [CLP for unstructured logs](../../core-unstructured/index) page. diff --git a/docs/src/user-guide/reference/reference-json-search-syntax.md b/docs/src/user-guide/reference-json-search-syntax.md similarity index 100% rename from docs/src/user-guide/reference/reference-json-search-syntax.md rename to docs/src/user-guide/reference-json-search-syntax.md diff --git a/docs/src/user-guide/reference/reference-text-search-syntax.md b/docs/src/user-guide/reference-text-search-syntax.md similarity index 100% rename from docs/src/user-guide/reference/reference-text-search-syntax.md rename to docs/src/user-guide/reference-text-search-syntax.md diff --git a/docs/src/user-guide/reference/reference-unstructured-schema-file.md b/docs/src/user-guide/reference-unstructured-schema-file.md similarity index 100% rename from docs/src/user-guide/reference/reference-unstructured-schema-file.md rename to docs/src/user-guide/reference-unstructured-schema-file.md diff --git a/docs/src/user-guide/reference/index.md b/docs/src/user-guide/reference/index.md deleted file mode 100644 index 5341702ca7..0000000000 --- a/docs/src/user-guide/reference/index.md +++ /dev/null @@ -1,23 +0,0 @@ -# Overview - -This section compiles information on syntax rules and format specifications relevant to CLP. - -::::{grid} 1 1 2 2 -:gutter: 2 - -:::{grid-item-card} -:link: reference-json-search-syntax -clp-json search syntax -::: - -:::{grid-item-card} -:link: reference-text-search-syntax -clp-text search syntax -::: - -:::{grid-item-card} -:link: reference-unstructured-schema-file -Schema file syntax -::: - -:::: diff --git a/docs/src/user-guide/resources/resources-datasets.md b/docs/src/user-guide/resources-datasets.md similarity index 100% rename from docs/src/user-guide/resources/resources-datasets.md rename to docs/src/user-guide/resources-datasets.md From 04f315eb27469d333ede570dae602d5ca5768381 Mon Sep 17 00:00:00 2001 From: quinntaylormitchell Date: Fri, 13 Jun 2025 12:47:43 -0400 Subject: [PATCH 15/40] Update docs/src/user-guide/guides/guides-overview.md Co-authored-by: Devin Gibson --- docs/src/user-guide/guides/guides-overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/user-guide/guides/guides-overview.md b/docs/src/user-guide/guides/guides-overview.md index c04e1dedc8..804b16adaa 100644 --- a/docs/src/user-guide/guides/guides-overview.md +++ b/docs/src/user-guide/guides/guides-overview.md @@ -16,6 +16,6 @@ Using CLP to ingest logs from object storage and store archives on object storag :link: guides-multi-node/multi-node Multi-node deployment ^^^ -Learn how to deploy CLP across multiple systems/nodes. +Learn how to deploy CLP across multiple nodes. ::: :::: From 721cc6ced65d8b15e2c324a361e28636f350ac99 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Fri, 13 Jun 2025 17:20:30 +0000 Subject: [PATCH 16/40] Small changes to address Devin's review --- docs/src/user-guide/quick-start/clp-json/main-page.md | 4 ++-- docs/src/user-guide/quick-start/clp-text/main-page.md | 2 +- docs/src/user-guide/quick-start/index.md | 6 +++--- docs/src/user-guide/quick-start/setup/main-page.md | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/src/user-guide/quick-start/clp-json/main-page.md b/docs/src/user-guide/quick-start/clp-json/main-page.md index a08b965b96..80af09a385 100644 --- a/docs/src/user-guide/quick-start/clp-json/main-page.md +++ b/docs/src/user-guide/quick-start/clp-json/main-page.md @@ -1,6 +1,6 @@ -# clp-json Quick-start +# clp-json quick-start -This page will walk you through how to start up CLP, and then use it to compress and search JSON logs. +This page will walk you through how to start up CLP and use it to compress and search JSON logs. :::{caution} If you're using the `clp-json` release, you can only compress JSON logs. diff --git a/docs/src/user-guide/quick-start/clp-text/main-page.md b/docs/src/user-guide/quick-start/clp-text/main-page.md index a378ed9b90..18f2921cfe 100644 --- a/docs/src/user-guide/quick-start/clp-text/main-page.md +++ b/docs/src/user-guide/quick-start/clp-text/main-page.md @@ -1,4 +1,4 @@ -# clp-text Quick-start +# clp-text quick-start This page will walk you through how to start up CLP, and then use it to compress and search unstructured text logs. diff --git a/docs/src/user-guide/quick-start/index.md b/docs/src/user-guide/quick-start/index.md index 79e140db23..6f61a5e8bd 100644 --- a/docs/src/user-guide/quick-start/index.md +++ b/docs/src/user-guide/quick-start/index.md @@ -13,7 +13,7 @@ View system requirements and choose a flavour of CLP. ::: :::: -If you already have CLP set up on your computer, click your flavour of CLP below to learn how to use it. +If you already have CLP set up, click your flavour of CLP below to learn how to use it. ::::{grid} 1 1 2 2 :gutter: 2 @@ -27,8 +27,8 @@ Learn about compressing and searching JSON logs. :::{grid-item-card} :link: clp-text/main-page -CLP for unstructured logs +CLP for unstructured text logs ^^^^^^^^^^^^ -Learn about compressing and searching unstructured logs. +Learn about compressing and searching unstructured text logs. ::: :::: diff --git a/docs/src/user-guide/quick-start/setup/main-page.md b/docs/src/user-guide/quick-start/setup/main-page.md index e132642d78..cf7f1b3939 100644 --- a/docs/src/user-guide/quick-start/setup/main-page.md +++ b/docs/src/user-guide/quick-start/setup/main-page.md @@ -1,6 +1,6 @@ # Setup -This page will guide you through CLP's system requirements, and help you choose which flavour of CLP will be best for you. +This page will guide you through CLP's system requirements and help you choose which flavour of CLP will be best for you. --- @@ -40,7 +40,7 @@ If you're planning to deploy CLP on multiple nodes/systems, there are a few othe ## Choosing a flavour -CLP has two flavours. Each flavour deals with a different type of log. +There are two flavours of CLP: * **[clp-json](#clp-json)** for compressing and searching **JSON** logs. * **[clp-text](#clp-text)** for compressing and searching **unstructured text** logs. From 5a89f4360098a58343ea79b96f41d603b81d3713 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Wed, 18 Jun 2025 20:27:35 +0000 Subject: [PATCH 17/40] Address Kirk's comments --- .../clp-json => }/clp-search-ui.png | 0 .../multi-node.md => guides-multi-node.md} | 6 ++-- .../{guides => }/guides-overview.md | 2 +- .../guides-using-object-storage/clp-config.md | 0 .../guides-using-object-storage/clp-usage.md | 2 +- .../guides-using-object-storage/index.md | 2 +- .../object-storage-config.md | 0 docs/src/user-guide/index.md | 17 +++++----- .../{clp-json/main-page.md => clp-json.md} | 10 +++--- .../{clp-text/main-page.md => clp-text.md} | 8 ++--- .../quick-start/clp-text/clp-search-ui.png | 3 -- docs/src/user-guide/quick-start/index.md | 34 ------------------- .../{setup/main-page.md => overview.md} | 28 ++++++++++++--- 13 files changed, 47 insertions(+), 65 deletions(-) rename docs/src/{user-guide/quick-start/clp-json => }/clp-search-ui.png (100%) rename docs/src/user-guide/{guides/guides-multi-node/multi-node.md => guides-multi-node.md} (96%) rename docs/src/user-guide/{guides => }/guides-overview.md (91%) rename docs/src/user-guide/{guides => }/guides-using-object-storage/clp-config.md (100%) rename docs/src/user-guide/{guides => }/guides-using-object-storage/clp-usage.md (97%) rename docs/src/user-guide/{guides => }/guides-using-object-storage/index.md (98%) rename docs/src/user-guide/{guides => }/guides-using-object-storage/object-storage-config.md (100%) rename docs/src/user-guide/quick-start/{clp-json/main-page.md => clp-json.md} (93%) rename docs/src/user-guide/quick-start/{clp-text/main-page.md => clp-text.md} (95%) delete mode 100644 docs/src/user-guide/quick-start/clp-text/clp-search-ui.png delete mode 100644 docs/src/user-guide/quick-start/index.md rename docs/src/user-guide/quick-start/{setup/main-page.md => overview.md} (89%) diff --git a/docs/src/user-guide/quick-start/clp-json/clp-search-ui.png b/docs/src/clp-search-ui.png similarity index 100% rename from docs/src/user-guide/quick-start/clp-json/clp-search-ui.png rename to docs/src/clp-search-ui.png diff --git a/docs/src/user-guide/guides/guides-multi-node/multi-node.md b/docs/src/user-guide/guides-multi-node.md similarity index 96% rename from docs/src/user-guide/guides/guides-multi-node/multi-node.md rename to docs/src/user-guide/guides-multi-node.md index ea6c461601..6799ddd397 100644 --- a/docs/src/user-guide/guides/guides-multi-node/multi-node.md +++ b/docs/src/user-guide/guides-multi-node.md @@ -110,20 +110,20 @@ Where `` is the name of the component in the groups above. ## Using CLP -Check out the [quick-start guide](../../quick-start/index) for your chosen flavour of CLP to learn how to compress and search your logs. +Check out the [quick-start guide](./quick-start/overview) for your chosen flavour of CLP to learn how to compress and search your logs. ::::{grid} 1 1 2 2 :gutter: 2 :::{grid-item-card} -:link: ../../quick-start/clp-json/main-page +:link: quick-start/clp-json clp-json ^^^ Compress and search JSON logs. ::: :::{grid-item-card} -:link: ../../quick-start/clp-text/main-page +:link: quick-start/clp-text clp-text ^^^ Compress and search unstructured text logs. diff --git a/docs/src/user-guide/guides/guides-overview.md b/docs/src/user-guide/guides-overview.md similarity index 91% rename from docs/src/user-guide/guides/guides-overview.md rename to docs/src/user-guide/guides-overview.md index 804b16adaa..9db20d4f56 100644 --- a/docs/src/user-guide/guides/guides-overview.md +++ b/docs/src/user-guide/guides-overview.md @@ -13,7 +13,7 @@ Using CLP to ingest logs from object storage and store archives on object storag ::: :::{grid-item-card} -:link: guides-multi-node/multi-node +:link: guides-multi-node Multi-node deployment ^^^ Learn how to deploy CLP across multiple nodes. diff --git a/docs/src/user-guide/guides/guides-using-object-storage/clp-config.md b/docs/src/user-guide/guides-using-object-storage/clp-config.md similarity index 100% rename from docs/src/user-guide/guides/guides-using-object-storage/clp-config.md rename to docs/src/user-guide/guides-using-object-storage/clp-config.md diff --git a/docs/src/user-guide/guides/guides-using-object-storage/clp-usage.md b/docs/src/user-guide/guides-using-object-storage/clp-usage.md similarity index 97% rename from docs/src/user-guide/guides/guides-using-object-storage/clp-usage.md rename to docs/src/user-guide/guides-using-object-storage/clp-usage.md index 15240cf7fe..e4eec83c76 100644 --- a/docs/src/user-guide/guides/guides-using-object-storage/clp-usage.md +++ b/docs/src/user-guide/guides-using-object-storage/clp-usage.md @@ -1,7 +1,7 @@ # Using CLP with object storage To compress logs from S3, follow the steps in the section below. For all other operations, you -should be able to use CLP as described in the [clp-json quick-start guide](../../quick-start/clp-json/main-page). +should be able to use CLP as described in the [clp-json quick-start guide](../quick-start/clp-json). ## Compressing logs from S3 diff --git a/docs/src/user-guide/guides/guides-using-object-storage/index.md b/docs/src/user-guide/guides-using-object-storage/index.md similarity index 98% rename from docs/src/user-guide/guides/guides-using-object-storage/index.md rename to docs/src/user-guide/guides-using-object-storage/index.md index 7db66516eb..90becbc110 100644 --- a/docs/src/user-guide/guides/guides-using-object-storage/index.md +++ b/docs/src/user-guide/guides-using-object-storage/index.md @@ -136,4 +136,4 @@ clp-usage [aws-iam-roles]: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html [aws-key-prefixes]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html [aws-sts-credentials]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html -[release-choices]: ../../quick-start/clp-json/main-page +[release-choices]: ../quick-start/clp-json diff --git a/docs/src/user-guide/guides/guides-using-object-storage/object-storage-config.md b/docs/src/user-guide/guides-using-object-storage/object-storage-config.md similarity index 100% rename from docs/src/user-guide/guides/guides-using-object-storage/object-storage-config.md rename to docs/src/user-guide/guides-using-object-storage/object-storage-config.md diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index ffc250ac41..a1e8bb6ff4 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -9,14 +9,14 @@ The sections are as follows: :gutter: 2 :::{grid-item-card} -:link: quick-start/index +:link: quick-start/overview Quick-start ^^^ A quick-start guide for choosing a flavour of CLP, setting it up, compressing your logs, and searching them. ::: :::{grid-item-card} -:link: guides/guides-overview +:link: guides-overview Guides ^^^ Guides for using CLP in various use cases. @@ -48,10 +48,9 @@ Reference docs like format specifications, etc. :hidden: :caption: Quick-start -quick-start/index -quick-start/setup/main-page -quick-start/clp-json/main-page -quick-start/clp-text/main-page +quick-start/overview +quick-start/clp-json +quick-start/clp-text ::: @@ -60,9 +59,9 @@ quick-start/clp-text/main-page :caption: Guides :glob: -guides/guides-overview -guides/guides-using-object-storage/index -guides/guides-multi-node/multi-node +guides-overview +guides-using-object-storage/index +guides-multi-node ::: diff --git a/docs/src/user-guide/quick-start/clp-json/main-page.md b/docs/src/user-guide/quick-start/clp-json.md similarity index 93% rename from docs/src/user-guide/quick-start/clp-json/main-page.md rename to docs/src/user-guide/quick-start/clp-json.md index 80af09a385..9384c2f305 100644 --- a/docs/src/user-guide/quick-start/clp-json/main-page.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -50,12 +50,12 @@ Compressed logs will be stored in the `/var/bin/archives` directory. :::{tip} To compress logs from object storage, see -[Using object storage](../../guides/guides-using-object-storage/index). +[Using object storage](../guides-using-object-storage/index). ::: ### Sample logs -For some sample logs, check out the open-source [datasets](../../resources-datasets). +For some sample logs, check out the open-source [datasets](../resources-datasets). ### Examining compression statistics @@ -82,7 +82,7 @@ where `key` is the name of the JSON key you'd like to search within, and `value` key1: value1 AND key2: value2 OR key3: value3 ... ``` -There are a number of other JSON-specific syntax rules that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference-json-search-syntax). +There are a number of other JSON-specific syntax rules that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../reference-json-search-syntax). ### Searching from the command line @@ -109,7 +109,7 @@ searches are case-**sensitive** on the command line. If you'd like to search your query from the web UI, CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). -:::{image} clp-search-ui.png +:::{image} ../../clp-search-ui.png ::: The image above shows the search page after running a query. The numbered circles correspond to the following features: @@ -145,4 +145,4 @@ sbin/stop-clp.sh ## More information -You've reached the end of the clp-json quick-start guide. For more information on clp-json, visit the [CLP for JSON logs](../../core-clp-s) page. +You've reached the end of the clp-json quick-start guide. For more information on clp-json, visit the [CLP for JSON logs](../core-clp-s) page. diff --git a/docs/src/user-guide/quick-start/clp-text/main-page.md b/docs/src/user-guide/quick-start/clp-text.md similarity index 95% rename from docs/src/user-guide/quick-start/clp-text/main-page.md rename to docs/src/user-guide/quick-start/clp-text.md index 18f2921cfe..66fb59ffe1 100644 --- a/docs/src/user-guide/quick-start/clp-text/main-page.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -37,7 +37,7 @@ Compressed logs will be stored in the `/var/bin/archives` directory. ### Sample logs -For some sample logs, check out the open-source [datasets](../../resources-datasets). +For some sample logs, check out the open-source [datasets](../resources-datasets). ### Examining compression statistics @@ -62,7 +62,7 @@ a*b?c would return all logs that contain the character `"a"` followed by any number (including zero) of other characters, followed by `"b"`, followed by one other character, followed by `"c"`. -There are a number of other syntax rules specific to unstructured text queries that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../../reference-text-search-syntax). +There are a number of other syntax rules specific to unstructured text queries that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../reference-text-search-syntax). ### Searching from the command line @@ -89,7 +89,7 @@ searches are case-**sensitive** on the command line. If you'd like to search your query from the web UI, CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). -:::{image} clp-search-ui.png +:::{image} ../../clp-search-ui.png ::: The image above shows the search page after running a query. The numbered circles correspond to the following features: @@ -125,4 +125,4 @@ sbin/stop-clp.sh ## More information -You've reached the end of the clp-text quick-start guide. For more information on clp-text, visit the [CLP for unstructured logs](../../core-unstructured/index) page. +You've reached the end of the clp-text quick-start guide. For more information on clp-text, visit the [CLP for unstructured logs](../core-unstructured/index) page. diff --git a/docs/src/user-guide/quick-start/clp-text/clp-search-ui.png b/docs/src/user-guide/quick-start/clp-text/clp-search-ui.png deleted file mode 100644 index 74530978a8..0000000000 --- a/docs/src/user-guide/quick-start/clp-text/clp-search-ui.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:80e2309f9b03856672edd73ae34dde67814ca04e2cd5235586b78a67fc4f18bf -size 57858 diff --git a/docs/src/user-guide/quick-start/index.md b/docs/src/user-guide/quick-start/index.md deleted file mode 100644 index 6f61a5e8bd..0000000000 --- a/docs/src/user-guide/quick-start/index.md +++ /dev/null @@ -1,34 +0,0 @@ -# Overview - -If you're new to CLP, follow the link below to learn how to set it up. - -::::{grid} 1 1 1 1 -:gutter: 2 - -:::{grid-item-card} -:link: setup/main-page -Setup -^^^ -View system requirements and choose a flavour of CLP. -::: -:::: - -If you already have CLP set up, click your flavour of CLP below to learn how to use it. - -::::{grid} 1 1 2 2 -:gutter: 2 - -:::{grid-item-card} -:link: clp-json/main-page -CLP for JSON logs -^^^ -Learn about compressing and searching JSON logs. -::: - -:::{grid-item-card} -:link: clp-text/main-page -CLP for unstructured text logs -^^^^^^^^^^^^ -Learn about compressing and searching unstructured text logs. -::: -:::: diff --git a/docs/src/user-guide/quick-start/setup/main-page.md b/docs/src/user-guide/quick-start/overview.md similarity index 89% rename from docs/src/user-guide/quick-start/setup/main-page.md rename to docs/src/user-guide/quick-start/overview.md index cf7f1b3939..ca56cf696e 100644 --- a/docs/src/user-guide/quick-start/setup/main-page.md +++ b/docs/src/user-guide/quick-start/overview.md @@ -1,7 +1,27 @@ -# Setup +# Overview This page will guide you through CLP's system requirements and help you choose which flavour of CLP will be best for you. +If you already have CLP set up, click your flavour of CLP below to learn how to use it. + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: clp-json +CLP for JSON logs +^^^ +Learn about compressing and searching JSON logs. +::: + +:::{grid-item-card} +:link: clp-text +CLP for unstructured text logs +^^^^^^^^^^^^ +Learn about compressing and searching unstructured text logs. +::: +:::: + --- ## System Requirements @@ -33,7 +53,7 @@ python3 --version If Python isn't on your system, or if the version isn't high enough, install or upgrade it. :::{note} -If you're planning to deploy CLP on multiple nodes/systems, there are a few other system requirements; check out the [multi-node deployment](../../guides/guides-multi-node/multi-node) page for more details. +If you're planning to deploy CLP on multiple nodes/systems, there are a few other system requirements; check out the [multi-node deployment](../guides-multi-node) page for more details. ::: --- @@ -80,7 +100,7 @@ If your logs are JSON logs, download and extract the JSON flavour tarball from t :gutter: 2 :::{grid-item-card} -:link: ../clp-json/main-page +:link: clp-json CLP for JSON logs ^^^ Learn how to start up clp-json, and begin compressing and searching JSON logs. @@ -118,7 +138,7 @@ If your logs are unstructured text logs, download and extract the unstructured t :gutter: 2 :::{grid-item-card} -:link: ../clp-text/main-page +:link: clp-text CLP for unstructured logs ^^^ Learn how to start up clp-text, and begin compressing and searching unstructured text logs. From 387a56dca1f7fc4fa292de148cf80a89494e1f9c Mon Sep 17 00:00:00 2001 From: quinntaylormitchell Date: Wed, 18 Jun 2025 16:32:46 -0400 Subject: [PATCH 18/40] Update docs/src/user-guide/quick-start/overview.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- docs/src/user-guide/quick-start/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/user-guide/quick-start/overview.md b/docs/src/user-guide/quick-start/overview.md index ca56cf696e..27c3fe6653 100644 --- a/docs/src/user-guide/quick-start/overview.md +++ b/docs/src/user-guide/quick-start/overview.md @@ -129,7 +129,7 @@ Caused by: java.net.UnknownHostException: i-e5d112ea ... 17 more ``` -The log file above contains two log events, both beginning with a timestamp. The first is a single +The log file above contains two log events, both beginning with a timestamp. The first is a single line, while the second contains multiple lines. line while the second contains multiple lines. If your logs are unstructured text logs, download and extract the unstructured text flavour tarball from the [Releases][clp-releases] page on GitHub, and proceed to the clp-text portal below. From e2f33044b4e78c8db7dc90ec917e1fd0eddba285 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Wed, 18 Jun 2025 20:41:51 +0000 Subject: [PATCH 19/40] Fix double line --- docs/src/user-guide/quick-start/overview.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user-guide/quick-start/overview.md b/docs/src/user-guide/quick-start/overview.md index 27c3fe6653..7d9bdb01cc 100644 --- a/docs/src/user-guide/quick-start/overview.md +++ b/docs/src/user-guide/quick-start/overview.md @@ -129,8 +129,8 @@ Caused by: java.net.UnknownHostException: i-e5d112ea ... 17 more ``` -The log file above contains two log events, both beginning with a timestamp. The first is a single line, while the second contains multiple lines. -line while the second contains multiple lines. +The log file above contains two log events, both beginning with a timestamp. The first is a single +line, while the second contains multiple lines. If your logs are unstructured text logs, download and extract the unstructured text flavour tarball from the [Releases][clp-releases] page on GitHub, and proceed to the clp-text portal below. From e46dc23d8a089ca55c09dc6332a0ed56891323f6 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Wed, 18 Jun 2025 21:39:31 +0000 Subject: [PATCH 20/40] Deduplicate content that's repeated between both clp-json and clp-text by using the include directive --- docs/conf/conf.py | 6 ++ docs/src/_shared/compression-stats.md | 8 +++ docs/src/_shared/sample-logs.md | 7 ++ docs/src/_shared/search-options.md | 50 ++++++++++++++ docs/src/_shared/start-clp.md | 16 +++++ docs/src/_shared/stop-clp.md | 11 +++ docs/src/user-guide/quick-start/clp-json.md | 76 ++------------------- docs/src/user-guide/quick-start/clp-text.md | 76 ++------------------- docs/src/user-guide/quick-start/overview.md | 2 +- 9 files changed, 113 insertions(+), 139 deletions(-) create mode 100644 docs/src/_shared/compression-stats.md create mode 100644 docs/src/_shared/sample-logs.md create mode 100644 docs/src/_shared/search-options.md create mode 100644 docs/src/_shared/start-clp.md create mode 100644 docs/src/_shared/stop-clp.md diff --git a/docs/conf/conf.py b/docs/conf/conf.py index 6cd36887ed..18e6f1a100 100644 --- a/docs/conf/conf.py +++ b/docs/conf/conf.py @@ -19,6 +19,12 @@ "sphinxcontrib.mermaid", ] +suppress_warnings = [ + "myst.header", + "image.not_readable", + "myst.xref_missing", +] + # -- MyST extensions ----------------------------------------------------------- # https://myst-parser.readthedocs.io/en/stable/syntax/optional.html myst_enable_extensions = [ diff --git a/docs/src/_shared/compression-stats.md b/docs/src/_shared/compression-stats.md new file mode 100644 index 0000000000..89e6eb13d0 --- /dev/null +++ b/docs/src/_shared/compression-stats.md @@ -0,0 +1,8 @@ +--- +orphan: true # keep Sphinx quiet about the ToC +--- + +### Examining compression statistics + +The compression script used above will output the compression ratio of each dataset you compress, or +you can use the UI to view overall statistics. diff --git a/docs/src/_shared/sample-logs.md b/docs/src/_shared/sample-logs.md new file mode 100644 index 0000000000..55c0fea92e --- /dev/null +++ b/docs/src/_shared/sample-logs.md @@ -0,0 +1,7 @@ +--- +orphan: true # keep Sphinx quiet about the ToC +--- + +### Sample logs + +For some sample logs, check out the open-source [datasets](../resources-datasets). diff --git a/docs/src/_shared/search-options.md b/docs/src/_shared/search-options.md new file mode 100644 index 0000000000..1d1ccec143 --- /dev/null +++ b/docs/src/_shared/search-options.md @@ -0,0 +1,50 @@ +--- +orphan: true # keep Sphinx quiet about the ToC +--- + +### Searching from the command line + +If you'd like to search your query from the command line, run the following command from inside the package: + +```bash +sbin/search.sh '' +``` + +To narrow your search to a specific time range: + +* Add `--begin-time ` to filter for log events after a certain time. + * `` is the timestamp as milliseconds since the UNIX epoch. +* Add `--end-time ` to filter for log events after a certain time. + +To perform case-insensitive searches, add the `--ignore-case` flag. + +:::{caution} +To match the convention of other tools, by default, searches are case-**insensitive** in the UI and +searches are case-**sensitive** on the command line. +::: + +### Searching from the UI + +If you'd like to search your query from the web UI, CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). + +:::{image} ../../clp-search-ui.png +::: + +The image above shows the search page after running a query. The numbered circles correspond to the following features: + +1. The search box is where you can enter your query. +2. The timeline shows the number of results across the time range of your query. + * You can click and drag to zoom into a time range or use the time range filter in (4). +3. The table displays the search results for your query. +4. Clicking the icon reveals additional filters for your query. + * The time range filter allows you to specify the period of time that matching log events must be + in. + * The case sensitivity filter allows you to specify whether CLP should respect the case of your + query. +5. Clicking the icon reveals options for displaying results. +6. The icon clears the results of the last query. + +:::{note} +By default, the UI will only return 1,000 of the latest search results. To perform searches which +return more results, use the [command line](#searching-from-the-command-line). +::: diff --git a/docs/src/_shared/start-clp.md b/docs/src/_shared/start-clp.md new file mode 100644 index 0000000000..f6d978726b --- /dev/null +++ b/docs/src/_shared/start-clp.md @@ -0,0 +1,16 @@ +--- +orphan: true # keep Sphinx quiet about the ToC +--- + +## Starting CLP + +To start CLP, run + +```bash +sbin/start-clp.sh +``` + +:::{note} +If CLP fails to start (e.g., due to a port conflict), try adjusting the settings in +`etc/clp-config.yml` and then run the start command again. +::: diff --git a/docs/src/_shared/stop-clp.md b/docs/src/_shared/stop-clp.md new file mode 100644 index 0000000000..9a715391e0 --- /dev/null +++ b/docs/src/_shared/stop-clp.md @@ -0,0 +1,11 @@ +--- +orphan: true # keep Sphinx quiet about the ToC +--- + +## Stopping CLP + +If you need to stop CLP, run the command + +```bash +sbin/stop-clp.sh +``` diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index 9384c2f305..e2b7832f8f 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -8,19 +8,9 @@ If you're using the `clp-json` release, you can only compress JSON logs. --- -## Starting CLP - -To start CLP, run - -```bash -sbin/start-clp.sh +```{include} ../../_shared/start-clp.md ``` -:::{note} -If CLP fails to start (e.g., due to a port conflict), try adjusting the settings in -`etc/clp-config.yml` and then run the start command again. -::: - --- ## Compressing JSON logs @@ -53,14 +43,11 @@ To compress logs from object storage, see [Using object storage](../guides-using-object-storage/index). ::: -### Sample logs - -For some sample logs, check out the open-source [datasets](../resources-datasets). - -### Examining compression statistics +```{include} ../../_shared/sample-logs.md +``` -The compression script used above will output the compression ratio of each dataset you compress, or -you can use the UI to view overall statistics. +```{include} ../../_shared/compression-stats.md +``` --- @@ -84,61 +71,12 @@ key1: value1 AND key2: value2 OR key3: value3 ... There are a number of other JSON-specific syntax rules that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../reference-json-search-syntax). -### Searching from the command line - -If you'd like to search your query from the command line, run the following command from inside the package: - -```bash -sbin/search.sh '' +```{include} ../../_shared/search-options.md ``` -To narrow your search to a specific time range: - -* Add `--begin-time ` to filter for log events after a certain time. - * `` is the timestamp as milliseconds since the UNIX epoch. -* Add `--end-time ` to filter for log events after a certain time. - -To perform case-insensitive searches, add the `--ignore-case` flag. - -:::{caution} -To match the convention of other tools, by default, searches are case-**insensitive** in the UI and -searches are case-**sensitive** on the command line. -::: - -### Searching from the UI - -If you'd like to search your query from the web UI, CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). - -:::{image} ../../clp-search-ui.png -::: - -The image above shows the search page after running a query. The numbered circles correspond to the following features: - -1. The search box is where you can enter your query. -2. The timeline shows the number of results across the time range of your query. - * You can click and drag to zoom into a time range or use the time range filter in (4). -3. The table displays the search results for your query. -4. Clicking the icon reveals additional filters for your query. - * The time range filter allows you to specify the period of time that matching log events must be - in. - * The case sensitivity filter allows you to specify whether CLP should respect the case of your - query. -5. Clicking the icon reveals options for displaying results. -6. The icon clears the results of the last query. - -:::{note} -By default, the UI will only return 1,000 of the latest search results. To perform searches which -return more results, use the [command line](#searching-from-the-command-line). -::: - --- -## Stopping CLP - -If you need to stop CLP, run the command - -```bash -sbin/stop-clp.sh +```{include} ../../_shared/stop-clp.md ``` --- diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index 66fb59ffe1..c85239ec37 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -8,19 +8,9 @@ If you're using the `clp-text` release, you should only compress unstructured te --- -## Starting CLP - -To start CLP, run - -```bash -sbin/start-clp.sh +```{include} ../../_shared/start-clp.md ``` -:::{note} -If CLP fails to start (e.g., due to a port conflict), try adjusting the settings in -`etc/clp-config.yml` and then run the start command again. -::: - --- ## Compressing unstructured text logs @@ -35,14 +25,11 @@ sbin/compress.sh [ ...] Compressed logs will be stored in the `/var/bin/archives` directory. -### Sample logs - -For some sample logs, check out the open-source [datasets](../resources-datasets). - -### Examining compression statistics +```{include} ../../_shared/sample-logs.md +``` -The compression script used above will output the compression ratio of each dataset you compress, or -you can use the UI to view overall statistics. +```{include} ../../_shared/compression-stats.md +``` --- @@ -64,61 +51,12 @@ would return all logs that contain the character `"a"` followed by any number (i There are a number of other syntax rules specific to unstructured text queries that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../reference-text-search-syntax). -### Searching from the command line - -If you'd like to search your query from the command line, run the following command from inside the package: - -```bash -sbin/search.sh '' +```{include} ../../_shared/search-options.md ``` -To narrow your search to a specific time range: - -* Add `--begin-time ` to filter for log events after a certain time. - * `` is the timestamp as milliseconds since the UNIX epoch. -* Add `--end-time ` to filter for log events after a certain time. - -To perform case-insensitive searches, add the `--ignore-case` flag. - -:::{caution} -To match the convention of other tools, by default, searches are case-**insensitive** in the UI and -searches are case-**sensitive** on the command line. -::: - -### Searching from the UI - -If you'd like to search your query from the web UI, CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). - -:::{image} ../../clp-search-ui.png -::: - -The image above shows the search page after running a query. The numbered circles correspond to the following features: - -1. The search box is where you can enter your query. -2. The timeline shows the number of results across the time range of your query. - * You can click and drag to zoom into a time range or use the time range filter in (4). -3. The table displays the search results for your query. -4. Clicking the icon reveals additional filters for your query. - * The time range filter allows you to specify the period of time that matching log events must be - in. - * The case sensitivity filter allows you to specify whether CLP should respect the case of your - query. -5. Clicking the icon reveals options for displaying results. -6. The icon clears the results of the last query. - -:::{note} -By default, the UI will only return 1,000 of the latest search results. To perform searches which -return more results, use the [command line](#searching-from-the-command-line). -::: - --- -## Stopping CLP - -If you need to stop CLP, run the command - -```bash -sbin/stop-clp.sh +```{include} ../../_shared/stop-clp.md ``` --- diff --git a/docs/src/user-guide/quick-start/overview.md b/docs/src/user-guide/quick-start/overview.md index 7d9bdb01cc..28aff51db6 100644 --- a/docs/src/user-guide/quick-start/overview.md +++ b/docs/src/user-guide/quick-start/overview.md @@ -139,7 +139,7 @@ If your logs are unstructured text logs, download and extract the unstructured t :::{grid-item-card} :link: clp-text -CLP for unstructured logs +CLP for unstructured text logs ^^^ Learn how to start up clp-text, and begin compressing and searching unstructured text logs. ::: From 22f0d89b5693f52f21dad94e4792ced50134bc77 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Wed, 18 Jun 2025 22:12:28 +0000 Subject: [PATCH 21/40] Address rabbit comments --- docs/conf/conf.py | 3 +++ docs/src/_shared/search-options.md | 2 +- docs/src/user-guide/core-container.md | 2 +- docs/src/user-guide/core-overview.md | 2 +- docs/src/user-guide/core-unstructured/index.md | 2 +- docs/src/user-guide/quick-start/clp-text.md | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/conf/conf.py b/docs/conf/conf.py index 18e6f1a100..fb1a7eb8da 100644 --- a/docs/conf/conf.py +++ b/docs/conf/conf.py @@ -19,6 +19,9 @@ "sphinxcontrib.mermaid", ] +# TODO: These suppressions are global, and while they solve the issue of suppressing +# the warnings re. using the {include} directive on the files in _shared (see PR#968), +# they may suppress legitimate warnings in the future. Find a better way to do this. suppress_warnings = [ "myst.header", "image.not_readable", diff --git a/docs/src/_shared/search-options.md b/docs/src/_shared/search-options.md index 1d1ccec143..488c396128 100644 --- a/docs/src/_shared/search-options.md +++ b/docs/src/_shared/search-options.md @@ -34,7 +34,7 @@ The image above shows the search page after running a query. The numbered circle 1. The search box is where you can enter your query. 2. The timeline shows the number of results across the time range of your query. - * You can click and drag to zoom into a time range or use the time range filter in (4). + * You can click and drag to zoom into a time range, or use the time range filter in (4). 3. The table displays the search results for your query. 4. Clicking the icon reveals additional filters for your query. * The time range filter allows you to specify the period of time that matching log events must be diff --git a/docs/src/user-guide/core-container.md b/docs/src/user-guide/core-container.md index e46b9ad6e9..cd78194c4f 100644 --- a/docs/src/user-guide/core-container.md +++ b/docs/src/user-guide/core-container.md @@ -30,7 +30,7 @@ docker run \ * Change `/my/data/dir` to the directory on your machine where you want to store the generated archives. It will be mounted at `/mnt/data` in the container. -Follow the usage instructions in [clp for unstructured logs](core-unstructured/index) or +Follow the usage instructions in [clp for unstructured text logs](core-unstructured/index) or [clp for JSON logs](core-clp-s), depending on the format of your logs. [1]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy diff --git a/docs/src/user-guide/core-overview.md b/docs/src/user-guide/core-overview.md index c8cdd9a312..c685903208 100644 --- a/docs/src/user-guide/core-overview.md +++ b/docs/src/user-guide/core-overview.md @@ -11,7 +11,7 @@ explain how to use each flavour. :::{grid-item-card} :link: core-unstructured/index -CLP for unstructured logs +CLP for unstructured text logs ^^^ A flavour of CLP for unstructured (e.g., free-text) logs. ::: diff --git a/docs/src/user-guide/core-unstructured/index.md b/docs/src/user-guide/core-unstructured/index.md index a39e54b339..bdeafca6c2 100644 --- a/docs/src/user-guide/core-unstructured/index.md +++ b/docs/src/user-guide/core-unstructured/index.md @@ -1,4 +1,4 @@ -# CLP for unstructured logs +# CLP for unstructured text logs There are two flavours of CLP for compressing unstructured logs. The guides below explain how to use each flavour. diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index c85239ec37..101975df54 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -63,4 +63,4 @@ There are a number of other syntax rules specific to unstructured text queries t ## More information -You've reached the end of the clp-text quick-start guide. For more information on clp-text, visit the [CLP for unstructured logs](../core-unstructured/index) page. +You've reached the end of the clp-text quick-start guide. For more information on clp-text, visit the [CLP for unstructured text logs](../core-unstructured/index) page. From 07997744a34b0663d0fd6e99d62422992f5490d6 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Wed, 18 Jun 2025 22:16:24 +0000 Subject: [PATCH 22/40] Lint --- docs/conf/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf/conf.py b/docs/conf/conf.py index fb1a7eb8da..1d847220e1 100644 --- a/docs/conf/conf.py +++ b/docs/conf/conf.py @@ -19,7 +19,7 @@ "sphinxcontrib.mermaid", ] -# TODO: These suppressions are global, and while they solve the issue of suppressing +# TODO: These suppressions are global, and while they solve the issue of suppressing # the warnings re. using the {include} directive on the files in _shared (see PR#968), # they may suppress legitimate warnings in the future. Find a better way to do this. suppress_warnings = [ From 07cde43ae0e07063909545266177e9db701b3654 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Mon, 23 Jun 2025 12:28:26 +0000 Subject: [PATCH 23/40] Restore content that had been deduplicated with 'include' command --- docs/conf/conf.py | 9 --- docs/src/_shared/compression-stats.md | 8 --- docs/src/_shared/sample-logs.md | 7 -- docs/src/_shared/search-options.md | 50 -------------- docs/src/_shared/start-clp.md | 16 ----- docs/src/_shared/stop-clp.md | 11 --- docs/src/user-guide/quick-start/clp-json.md | 76 +++++++++++++++++++-- docs/src/user-guide/quick-start/clp-text.md | 76 +++++++++++++++++++-- 8 files changed, 138 insertions(+), 115 deletions(-) delete mode 100644 docs/src/_shared/compression-stats.md delete mode 100644 docs/src/_shared/sample-logs.md delete mode 100644 docs/src/_shared/search-options.md delete mode 100644 docs/src/_shared/start-clp.md delete mode 100644 docs/src/_shared/stop-clp.md diff --git a/docs/conf/conf.py b/docs/conf/conf.py index 1d847220e1..6cd36887ed 100644 --- a/docs/conf/conf.py +++ b/docs/conf/conf.py @@ -19,15 +19,6 @@ "sphinxcontrib.mermaid", ] -# TODO: These suppressions are global, and while they solve the issue of suppressing -# the warnings re. using the {include} directive on the files in _shared (see PR#968), -# they may suppress legitimate warnings in the future. Find a better way to do this. -suppress_warnings = [ - "myst.header", - "image.not_readable", - "myst.xref_missing", -] - # -- MyST extensions ----------------------------------------------------------- # https://myst-parser.readthedocs.io/en/stable/syntax/optional.html myst_enable_extensions = [ diff --git a/docs/src/_shared/compression-stats.md b/docs/src/_shared/compression-stats.md deleted file mode 100644 index 89e6eb13d0..0000000000 --- a/docs/src/_shared/compression-stats.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -orphan: true # keep Sphinx quiet about the ToC ---- - -### Examining compression statistics - -The compression script used above will output the compression ratio of each dataset you compress, or -you can use the UI to view overall statistics. diff --git a/docs/src/_shared/sample-logs.md b/docs/src/_shared/sample-logs.md deleted file mode 100644 index 55c0fea92e..0000000000 --- a/docs/src/_shared/sample-logs.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -orphan: true # keep Sphinx quiet about the ToC ---- - -### Sample logs - -For some sample logs, check out the open-source [datasets](../resources-datasets). diff --git a/docs/src/_shared/search-options.md b/docs/src/_shared/search-options.md deleted file mode 100644 index 488c396128..0000000000 --- a/docs/src/_shared/search-options.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -orphan: true # keep Sphinx quiet about the ToC ---- - -### Searching from the command line - -If you'd like to search your query from the command line, run the following command from inside the package: - -```bash -sbin/search.sh '' -``` - -To narrow your search to a specific time range: - -* Add `--begin-time ` to filter for log events after a certain time. - * `` is the timestamp as milliseconds since the UNIX epoch. -* Add `--end-time ` to filter for log events after a certain time. - -To perform case-insensitive searches, add the `--ignore-case` flag. - -:::{caution} -To match the convention of other tools, by default, searches are case-**insensitive** in the UI and -searches are case-**sensitive** on the command line. -::: - -### Searching from the UI - -If you'd like to search your query from the web UI, CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). - -:::{image} ../../clp-search-ui.png -::: - -The image above shows the search page after running a query. The numbered circles correspond to the following features: - -1. The search box is where you can enter your query. -2. The timeline shows the number of results across the time range of your query. - * You can click and drag to zoom into a time range, or use the time range filter in (4). -3. The table displays the search results for your query. -4. Clicking the icon reveals additional filters for your query. - * The time range filter allows you to specify the period of time that matching log events must be - in. - * The case sensitivity filter allows you to specify whether CLP should respect the case of your - query. -5. Clicking the icon reveals options for displaying results. -6. The icon clears the results of the last query. - -:::{note} -By default, the UI will only return 1,000 of the latest search results. To perform searches which -return more results, use the [command line](#searching-from-the-command-line). -::: diff --git a/docs/src/_shared/start-clp.md b/docs/src/_shared/start-clp.md deleted file mode 100644 index f6d978726b..0000000000 --- a/docs/src/_shared/start-clp.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -orphan: true # keep Sphinx quiet about the ToC ---- - -## Starting CLP - -To start CLP, run - -```bash -sbin/start-clp.sh -``` - -:::{note} -If CLP fails to start (e.g., due to a port conflict), try adjusting the settings in -`etc/clp-config.yml` and then run the start command again. -::: diff --git a/docs/src/_shared/stop-clp.md b/docs/src/_shared/stop-clp.md deleted file mode 100644 index 9a715391e0..0000000000 --- a/docs/src/_shared/stop-clp.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -orphan: true # keep Sphinx quiet about the ToC ---- - -## Stopping CLP - -If you need to stop CLP, run the command - -```bash -sbin/stop-clp.sh -``` diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index e2b7832f8f..ab2be0f649 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -8,9 +8,19 @@ If you're using the `clp-json` release, you can only compress JSON logs. --- -```{include} ../../_shared/start-clp.md +## Starting CLP + +To start CLP, run + +```bash +sbin/start-clp.sh ``` +:::{note} +If CLP fails to start (e.g., due to a port conflict), try adjusting the settings in +`etc/clp-config.yml` and then run the start command again. +::: + --- ## Compressing JSON logs @@ -43,11 +53,14 @@ To compress logs from object storage, see [Using object storage](../guides-using-object-storage/index). ::: -```{include} ../../_shared/sample-logs.md -``` +### Sample logs -```{include} ../../_shared/compression-stats.md -``` +For some sample logs, check out the open-source [datasets](../resources-datasets). + +### Examining compression statistics + +The compression script used above will output the compression ratio of each dataset you compress, or +you can use the UI to view overall statistics. --- @@ -71,12 +84,61 @@ key1: value1 AND key2: value2 OR key3: value3 ... There are a number of other JSON-specific syntax rules that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../reference-json-search-syntax). -```{include} ../../_shared/search-options.md +### Searching from the command line + +If you'd like to search your query from the command line, run the following command from inside the package: + +```bash +sbin/search.sh '' ``` +To narrow your search to a specific time range: + +* Add `--begin-time ` to filter for log events after a certain time. + * `` is the timestamp as milliseconds since the UNIX epoch. +* Add `--end-time ` to filter for log events after a certain time. + +To perform case-insensitive searches, add the `--ignore-case` flag. + +:::{caution} +To match the convention of other tools, by default, searches are case-**insensitive** in the UI and +searches are case-**sensitive** on the command line. +::: + +### Searching from the UI + +If you'd like to search your query from the web UI, CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). + +:::{image} ../../clp-search-ui.png +::: + +The image above shows the search page after running a query. The numbered circles correspond to the following features: + +1. The search box is where you can enter your query. +2. The timeline shows the number of results across the time range of your query. + * You can click and drag to zoom into a time range, or use the time range filter in (4). +3. The table displays the search results for your query. +4. Clicking the icon reveals additional filters for your query. + * The time range filter allows you to specify the period of time that matching log events must be + in. + * The case sensitivity filter allows you to specify whether CLP should respect the case of your + query. +5. Clicking the icon reveals options for displaying results. +6. The icon clears the results of the last query. + +:::{note} +By default, the UI will only return 1,000 of the latest search results. To perform searches which +return more results, use the [command line](#searching-from-the-command-line). +::: + --- -```{include} ../../_shared/stop-clp.md +## Stopping CLP + +If you need to stop CLP, run the command + +```bash +sbin/stop-clp.sh ``` --- diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index 101975df54..3edb532a51 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -8,9 +8,19 @@ If you're using the `clp-text` release, you should only compress unstructured te --- -```{include} ../../_shared/start-clp.md +## Starting CLP + +To start CLP, run + +```bash +sbin/start-clp.sh ``` +:::{note} +If CLP fails to start (e.g., due to a port conflict), try adjusting the settings in +`etc/clp-config.yml` and then run the start command again. +::: + --- ## Compressing unstructured text logs @@ -25,11 +35,14 @@ sbin/compress.sh [ ...] Compressed logs will be stored in the `/var/bin/archives` directory. -```{include} ../../_shared/sample-logs.md -``` +### Sample logs -```{include} ../../_shared/compression-stats.md -``` +For some sample logs, check out the open-source [datasets](../resources-datasets). + +### Examining compression statistics + +The compression script used above will output the compression ratio of each dataset you compress, or +you can use the UI to view overall statistics. --- @@ -51,12 +64,61 @@ would return all logs that contain the character `"a"` followed by any number (i There are a number of other syntax rules specific to unstructured text queries that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../reference-text-search-syntax). -```{include} ../../_shared/search-options.md +### Searching from the command line + +If you'd like to search your query from the command line, run the following command from inside the package: + +```bash +sbin/search.sh '' ``` +To narrow your search to a specific time range: + +* Add `--begin-time ` to filter for log events after a certain time. + * `` is the timestamp as milliseconds since the UNIX epoch. +* Add `--end-time ` to filter for log events after a certain time. + +To perform case-insensitive searches, add the `--ignore-case` flag. + +:::{caution} +To match the convention of other tools, by default, searches are case-**insensitive** in the UI and +searches are case-**sensitive** on the command line. +::: + +### Searching from the UI + +If you'd like to search your query from the web UI, CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). + +:::{image} ../../clp-search-ui.png +::: + +The image above shows the search page after running a query. The numbered circles correspond to the following features: + +1. The search box is where you can enter your query. +2. The timeline shows the number of results across the time range of your query. + * You can click and drag to zoom into a time range, or use the time range filter in (4). +3. The table displays the search results for your query. +4. Clicking the icon reveals additional filters for your query. + * The time range filter allows you to specify the period of time that matching log events must be + in. + * The case sensitivity filter allows you to specify whether CLP should respect the case of your + query. +5. Clicking the icon reveals options for displaying results. +6. The icon clears the results of the last query. + +:::{note} +By default, the UI will only return 1,000 of the latest search results. To perform searches which +return more results, use the [command line](#searching-from-the-command-line). +::: + --- -```{include} ../../_shared/stop-clp.md +## Stopping CLP + +If you need to stop CLP, run the command + +```bash +sbin/stop-clp.sh ``` --- From 5ba7a0f46d7d12fc18d61404b85ea9231732f609 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Mon, 23 Jun 2025 12:53:39 +0000 Subject: [PATCH 24/40] Address Kirk's comments --- docs/src/user-guide/core-container.md | 2 +- docs/src/user-guide/core-overview.md | 2 +- .../src/user-guide/core-unstructured/index.md | 2 +- docs/src/user-guide/guides-multi-node.md | 2 +- docs/src/user-guide/guides-overview.md | 2 +- docs/src/user-guide/index.md | 10 +-- .../quick-start/{overview.md => index.md} | 78 ++++++++----------- 7 files changed, 41 insertions(+), 57 deletions(-) rename docs/src/user-guide/quick-start/{overview.md => index.md} (56%) diff --git a/docs/src/user-guide/core-container.md b/docs/src/user-guide/core-container.md index cd78194c4f..e46b9ad6e9 100644 --- a/docs/src/user-guide/core-container.md +++ b/docs/src/user-guide/core-container.md @@ -30,7 +30,7 @@ docker run \ * Change `/my/data/dir` to the directory on your machine where you want to store the generated archives. It will be mounted at `/mnt/data` in the container. -Follow the usage instructions in [clp for unstructured text logs](core-unstructured/index) or +Follow the usage instructions in [clp for unstructured logs](core-unstructured/index) or [clp for JSON logs](core-clp-s), depending on the format of your logs. [1]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy diff --git a/docs/src/user-guide/core-overview.md b/docs/src/user-guide/core-overview.md index c685903208..c8cdd9a312 100644 --- a/docs/src/user-guide/core-overview.md +++ b/docs/src/user-guide/core-overview.md @@ -11,7 +11,7 @@ explain how to use each flavour. :::{grid-item-card} :link: core-unstructured/index -CLP for unstructured text logs +CLP for unstructured logs ^^^ A flavour of CLP for unstructured (e.g., free-text) logs. ::: diff --git a/docs/src/user-guide/core-unstructured/index.md b/docs/src/user-guide/core-unstructured/index.md index bdeafca6c2..a39e54b339 100644 --- a/docs/src/user-guide/core-unstructured/index.md +++ b/docs/src/user-guide/core-unstructured/index.md @@ -1,4 +1,4 @@ -# CLP for unstructured text logs +# CLP for unstructured logs There are two flavours of CLP for compressing unstructured logs. The guides below explain how to use each flavour. diff --git a/docs/src/user-guide/guides-multi-node.md b/docs/src/user-guide/guides-multi-node.md index 6799ddd397..5aad660946 100644 --- a/docs/src/user-guide/guides-multi-node.md +++ b/docs/src/user-guide/guides-multi-node.md @@ -110,7 +110,7 @@ Where `` is the name of the component in the groups above. ## Using CLP -Check out the [quick-start guide](./quick-start/overview) for your chosen flavour of CLP to learn how to compress and search your logs. +Check out the [quick-start guide](./quick-start/index) for your chosen flavour of CLP to learn how to compress and search your logs. ::::{grid} 1 1 2 2 :gutter: 2 diff --git a/docs/src/user-guide/guides-overview.md b/docs/src/user-guide/guides-overview.md index 9db20d4f56..02faefe2a1 100644 --- a/docs/src/user-guide/guides-overview.md +++ b/docs/src/user-guide/guides-overview.md @@ -16,6 +16,6 @@ Using CLP to ingest logs from object storage and store archives on object storag :link: guides-multi-node Multi-node deployment ^^^ -Learn how to deploy CLP across multiple nodes. +How to deploy CLP across multiple nodes. ::: :::: diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index a1e8bb6ff4..1df3d77b7e 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -9,8 +9,8 @@ The sections are as follows: :gutter: 2 :::{grid-item-card} -:link: quick-start/overview -Quick-start +:link: quick-start/index +Quick start ^^^ A quick-start guide for choosing a flavour of CLP, setting it up, compressing your logs, and searching them. ::: @@ -46,12 +46,11 @@ Reference docs like format specifications, etc. :::{toctree} :hidden: -:caption: Quick-start +:caption: Quick start -quick-start/overview +quick-start/index quick-start/clp-json quick-start/clp-text - ::: :::{toctree} @@ -62,7 +61,6 @@ quick-start/clp-text guides-overview guides-using-object-storage/index guides-multi-node - ::: :::{toctree} diff --git a/docs/src/user-guide/quick-start/overview.md b/docs/src/user-guide/quick-start/index.md similarity index 56% rename from docs/src/user-guide/quick-start/overview.md rename to docs/src/user-guide/quick-start/index.md index 28aff51db6..c5d3a26189 100644 --- a/docs/src/user-guide/quick-start/overview.md +++ b/docs/src/user-guide/quick-start/index.md @@ -1,60 +1,44 @@ # Overview -This page will guide you through CLP's system requirements and help you choose which flavour of CLP will be best for you. +This guide describes: -If you already have CLP set up, click your flavour of CLP below to learn how to use it. - -::::{grid} 1 1 2 2 -:gutter: 2 - -:::{grid-item-card} -:link: clp-json -CLP for JSON logs -^^^ -Learn about compressing and searching JSON logs. -::: - -:::{grid-item-card} -:link: clp-text -CLP for unstructured text logs -^^^^^^^^^^^^ -Learn about compressing and searching unstructured text logs. -::: -:::: +- [CLP's system requirements](#system-requirements); +- [how to choose a CLP flavor](#choosing-a-flavour); and +- [how to use CLP](#using-clp). --- ## System Requirements -To run properly on your system, CLP requires a few other programs. Before you set up CLP, ensure that you have the following installed. +To run a CLP release, you'll need: + +* [Docker](#docker) +* [Python](#python) ### Docker -CLP uses Docker to deploy its different components. You can check whether Docker is already installed on your system by running the command +To check whether Docker is installed on your system, run: ```bash docker version ``` -If Docker is not on your system, follow the instructions [here][Docker] to install it. +If Docker isn't installed, follow these [instructions][Docker] to install it. -If you're not running as root, ensure Docker can be run [without superuser privileges][docker-non-root]. +NOTE: -If you're using Docker Desktop, ensure version 4.34 or higher is installed, and [host networking is enabled][docker-desktop-host-networking]. +* If you're not running as root, ensure Docker can be run [without superuser privileges][docker-non-root]. +* If you're using Docker Desktop, ensure version 4.34 or higher is installed, and [host networking is enabled][docker-desktop-host-networking]. ### Python -CLP uses Python to interpret the scripts that coordinate how it runs. Specifically, CLP needs Python version 3.8 or higher. Many Linux distributions come with Python pre-installed; to confirm that it's on your system, run the command +To check whether Python is installed on your system, run: ```bash python3 --version ``` -If Python isn't on your system, or if the version isn't high enough, install or upgrade it. - -:::{note} -If you're planning to deploy CLP on multiple nodes/systems, there are a few other system requirements; check out the [multi-node deployment](../guides-multi-node) page for more details. -::: +CLP requires Python 3.8 or higher. If Python isn't installed, or if the version isn't high enough, install or upgrade it by following the instructions for your OS. --- @@ -94,18 +78,7 @@ object. For example: The log file above contains two log events represented by two JSON objects printed one after the other. Whitespace is ignored, so the log events could also appear with no newlines and indentation. -If your logs are JSON logs, download and extract the JSON flavour tarball from the [Releases][clp-releases] page on GitHub, and proceed to the clp-json portal below. - -::::{grid} 1 1 1 1 -:gutter: 2 - -:::{grid-item-card} -:link: clp-json -CLP for JSON logs -^^^ -Learn how to start up clp-json, and begin compressing and searching JSON logs. -::: -:::: +If you're using JSON logs, download and extract the `clp-json` release from the [Releases][clp-releases] page, then process to the [clp-json](./clp-json.md) quick-start guide. ### clp-text @@ -132,16 +105,29 @@ Caused by: java.net.UnknownHostException: i-e5d112ea The log file above contains two log events, both beginning with a timestamp. The first is a single line, while the second contains multiple lines. -If your logs are unstructured text logs, download and extract the unstructured text flavour tarball from the [Releases][clp-releases] page on GitHub, and proceed to the clp-text portal below. +If you're using unstructured text logs, download and extract the `clp-text` release from the [Releases][clp-releases] page, then process to the [clp-text](./clp-text.md) quick-start guide. + +--- + +## Using CLP -::::{grid} 1 1 1 1 +Once you have CLP set up, proceed to the quick start guide for your chosen flavour by clicking the link below. + +::::{grid} 1 1 2 2 :gutter: 2 +:::{grid-item-card} +:link: clp-json +CLP for JSON logs +^^^ +Learn about compressing and searching JSON logs. +::: + :::{grid-item-card} :link: clp-text CLP for unstructured text logs ^^^ -Learn how to start up clp-text, and begin compressing and searching unstructured text logs. +Learn about compressing and searching unstructured text logs. ::: :::: From 8ccbaf0b916113dd731c6f393355c4ec06fd3a6e Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Mon, 23 Jun 2025 13:31:07 +0000 Subject: [PATCH 25/40] Use American English spelling (only modified files in quick-start folder; leave other spelling changes for another PR) --- docs/src/user-guide/quick-start/index.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/src/user-guide/quick-start/index.md b/docs/src/user-guide/quick-start/index.md index c5d3a26189..16672fa13c 100644 --- a/docs/src/user-guide/quick-start/index.md +++ b/docs/src/user-guide/quick-start/index.md @@ -3,7 +3,7 @@ This guide describes: - [CLP's system requirements](#system-requirements); -- [how to choose a CLP flavor](#choosing-a-flavour); and +- [how to choose a CLP flavor](#choosing-a-flavor); and - [how to use CLP](#using-clp). --- @@ -42,21 +42,21 @@ CLP requires Python 3.8 or higher. If Python isn't installed, or if the version --- -## Choosing a flavour +## Choosing a flavor -There are two flavours of CLP: +There are two flavors of CLP: * **[clp-json](#clp-json)** for compressing and searching **JSON** logs. * **[clp-text](#clp-text)** for compressing and searching **unstructured text** logs. :::{note} -Both flavours contain the same binaries but are configured with different values for the +Both flavors contain the same binaries but are configured with different values for the `package.storage_engine` key in the package's config file (`etc/clp-config.yml`). ::: ### clp-json -The JSON flavour of CLP is appropriate for JSON logs, where each log event is an independent JSON +The JSON flavor of CLP is appropriate for JSON logs, where each log event is an independent JSON object. For example: ```json lines @@ -82,7 +82,7 @@ If you're using JSON logs, download and extract the `clp-json` release from the ### clp-text -The unstructured text flavour of CLP is appropriate for unstructured text logs, where each log event contains a +The unstructured text flavor of CLP is appropriate for unstructured text logs, where each log event contains a timestamp and may span one or more lines. :::{note} @@ -111,7 +111,7 @@ If you're using unstructured text logs, download and extract the `clp-text` rele ## Using CLP -Once you have CLP set up, proceed to the quick start guide for your chosen flavour by clicking the link below. +Once you have CLP set up, proceed to the quick start guide for your chosen flavor by clicking the link below. ::::{grid} 1 1 2 2 :gutter: 2 From 1e8b6a5313eaac11312b6a337d0882b895f815b8 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Mon, 23 Jun 2025 13:53:52 +0000 Subject: [PATCH 26/40] Change formatting to comply with markdownlintcli2 --- .../guides-using-object-storage/index.md | 2 +- docs/src/user-guide/quick-start/clp-json.md | 42 ++++++++++++------- docs/src/user-guide/quick-start/clp-text.md | 36 +++++++++++----- docs/src/user-guide/quick-start/index.md | 30 ++++++++----- 4 files changed, 71 insertions(+), 39 deletions(-) diff --git a/docs/src/user-guide/guides-using-object-storage/index.md b/docs/src/user-guide/guides-using-object-storage/index.md index 90becbc110..92e3da90d3 100644 --- a/docs/src/user-guide/guides-using-object-storage/index.md +++ b/docs/src/user-guide/guides-using-object-storage/index.md @@ -11,7 +11,7 @@ to use object storage for any combination of the three use cases (e.g., compress cache the stream files on S3, but store archives on the local filesystem). :::{note} -Currently, only the [clp-json][release-choices] flavour supports object storage. Support for +Currently, only the [clp-json][release-choices] flavor supports object storage. Support for `clp-text` will be added in a future release. ::: diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index ab2be0f649..d77f641e8b 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -32,19 +32,19 @@ sbin/compress.sh fs --timestamp-key '' [ ...] ``` * `` is the field path of the kv-pair that contains the timestamp in each log event. - * E.g., if your log events look like - `{"timestamp": {"iso8601": "2024-01-01 00:01:02.345", ...}}`, you should enter - `timestamp.iso8601` as the timestamp key. + * E.g., if your log events look like + `{"timestamp": {"iso8601": "2024-01-01 00:01:02.345", ...}}`, you should enter + `timestamp.iso8601` as the timestamp key. :::{caution} - Log events without the specified timestamp key will _not_ have an assigned timestamp. Currently, + Log events without the specified timestamp key will *not* have an assigned timestamp. Currently, these events can only be searched from the command line (when you don't specify a timestamp filter). ::: * `` are paths to JSON log files or directories containing such files. - * Each JSON log file should contain each log event as a separate JSON object, - i.e., _not_ as an array. + * Each JSON log file should contain each log event as a separate JSON object, + i.e., *not* as an array. Compressed logs will be stored in the `/var/bin/archives` directory. @@ -59,8 +59,8 @@ For some sample logs, check out the open-source [datasets](../resources-datasets ### Examining compression statistics -The compression script used above will output the compression ratio of each dataset you compress, or -you can use the UI to view overall statistics. +The compression script used above will output the compression ratio of each dataset you compress, +or you can use the UI to view overall statistics. --- @@ -70,23 +70,29 @@ You can search through your logs using queries from the UI or from the command l ### Queries -Regardless of what method you use to search, you'll need a query to find the logs you're looking for. Queries for JSON logs take the general form of +Regardless of what method you use to search, you'll need a query to find the logs you're looking +for. Queries for JSON logs take the general form of ```bash key: value ``` -where `key` is the name of the JSON key you'd like to search within, and `value` is a sequence of characters you're looking for within that key. Multiple key-value pairs can be chained together with `AND`, `OR`, or `NOT`, like so: +where `key` is the name of the JSON key you'd like to search within, and `value` is a sequence of +characters you're looking for within that key. Multiple key-value pairs can be chained together +with `AND`, `OR`, or `NOT`, like so: ```bash key1: value1 AND key2: value2 OR key3: value3 ... ``` -There are a number of other JSON-specific syntax rules that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../reference-json-search-syntax). +There are a number of other JSON-specific syntax rules that you can use to make your searches more +powerful and effective. You can read about these rules on the +[JSON syntax reference page](../reference-json-search-syntax). ### Searching from the command line -If you'd like to search your query from the command line, run the following command from inside the package: +If you'd like to search your query from the command line, run the following command from inside the +package: ```bash sbin/search.sh '' @@ -95,7 +101,7 @@ sbin/search.sh '' To narrow your search to a specific time range: * Add `--begin-time ` to filter for log events after a certain time. - * `` is the timestamp as milliseconds since the UNIX epoch. + * `` is the timestamp as milliseconds since the UNIX epoch. * Add `--end-time ` to filter for log events after a certain time. To perform case-insensitive searches, add the `--ignore-case` flag. @@ -107,12 +113,15 @@ searches are case-**sensitive** on the command line. ### Searching from the UI -If you'd like to search your query from the web UI, CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). +If you'd like to search your query from the web UI, CLP includes a web interface available at +[http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or +`webui.port` in `etc/clp-config.yml`, use the new values). :::{image} ../../clp-search-ui.png ::: -The image above shows the search page after running a query. The numbered circles correspond to the following features: +The image above shows the search page after running a query. The numbered circles correspond to +the following features: 1. The search box is where you can enter your query. 2. The timeline shows the number of results across the time range of your query. @@ -145,4 +154,5 @@ sbin/stop-clp.sh ## More information -You've reached the end of the clp-json quick-start guide. For more information on clp-json, visit the [CLP for JSON logs](../core-clp-s) page. +You've reached the end of the clp-json quick-start guide. For more information on clp-json, +visit the [CLP for JSON logs](../core-clp-s) page. diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index 3edb532a51..e2b34a5c04 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -1,9 +1,12 @@ # clp-text quick-start -This page will walk you through how to start up CLP, and then use it to compress and search unstructured text logs. +This page will walk you through how to start up CLP, and then use it to compress and search +unstructured text logs. :::{caution} -If you're using the `clp-text` release, you should only compress unstructured text logs. `clp-text` is able to compress and search JSON logs as if it was unstructured text, but `clp-text` cannot query individual fields. This limitation will be addressed in a future version of CLP. +If you're using the `clp-text` release, you should only compress unstructured text logs. `clp-text` +is able to compress and search JSON logs as if it was unstructured text, but `clp-text` cannot +query individual fields. This limitation will be addressed in a future version of CLP. ::: --- @@ -52,21 +55,28 @@ You can search through your logs using queries from the UI or from the command l ### Queries -Regardless of what method you use to search, you'll need a query to find the logs you're looking for. All unstructured text log queries are written as plain text. +Regardless of what method you use to search, you'll need a query to find the logs you're looking +for. All unstructured text log queries are written as plain text. -You can use a couple of special characters to make these queries more versatile. `*` can be used as a placeholder for an unknown number of characters, and `?` can be used for a single character. For example, the query +You can use a couple of special characters to make these queries more versatile. `*` can be used as +a placeholder for an unknown number of characters, and `?` can be used for a single character. +For example, the query ```bash a*b?c ``` -would return all logs that contain the character `"a"` followed by any number (including zero) of other characters, followed by `"b"`, followed by one other character, followed by `"c"`. +would return all logs that contain the character `"a"` followed by any number (including zero) of +other characters, followed by `"b"`, followed by one other character, followed by `"c"`. -There are a number of other syntax rules specific to unstructured text queries that you can use to make your searches more powerful and effective. You can read about these syntax rules [here](../reference-text-search-syntax). +There are a number of other syntax rules specific to unstructured text queries that you can use to +make your searches more powerful and effective. You can read about these rules on the +[text syntax reference page](../reference-text-search-syntax). ### Searching from the command line -If you'd like to search your query from the command line, run the following command from inside the package: +If you'd like to search your query from the command line, run the following command from inside the +package: ```bash sbin/search.sh '' @@ -75,7 +85,7 @@ sbin/search.sh '' To narrow your search to a specific time range: * Add `--begin-time ` to filter for log events after a certain time. - * `` is the timestamp as milliseconds since the UNIX epoch. + * `` is the timestamp as milliseconds since the UNIX epoch. * Add `--end-time ` to filter for log events after a certain time. To perform case-insensitive searches, add the `--ignore-case` flag. @@ -87,12 +97,15 @@ searches are case-**sensitive** on the command line. ### Searching from the UI -If you'd like to search your query from the web UI, CLP includes a web interface available at [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). +If you'd like to search your query from the web UI, CLP includes a web interface available at +[http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or +`webui.port` in `etc/clp-config.yml`, use the new values). :::{image} ../../clp-search-ui.png ::: -The image above shows the search page after running a query. The numbered circles correspond to the following features: +The image above shows the search page after running a query. The numbered circles correspond to the +following features: 1. The search box is where you can enter your query. 2. The timeline shows the number of results across the time range of your query. @@ -125,4 +138,5 @@ sbin/stop-clp.sh ## More information -You've reached the end of the clp-text quick-start guide. For more information on clp-text, visit the [CLP for unstructured text logs](../core-unstructured/index) page. +You've reached the end of the clp-text quick-start guide. For more information on clp-text, +visit the [CLP for unstructured text logs](../core-unstructured/index) page. diff --git a/docs/src/user-guide/quick-start/index.md b/docs/src/user-guide/quick-start/index.md index 16672fa13c..5ac225f2f5 100644 --- a/docs/src/user-guide/quick-start/index.md +++ b/docs/src/user-guide/quick-start/index.md @@ -2,9 +2,9 @@ This guide describes: -- [CLP's system requirements](#system-requirements); -- [how to choose a CLP flavor](#choosing-a-flavor); and -- [how to use CLP](#using-clp). +* [CLP's system requirements](#system-requirements); +* [how to choose a CLP flavor](#choosing-a-flavor); and +* [how to use CLP](#using-clp). --- @@ -27,8 +27,10 @@ If Docker isn't installed, follow these [instructions][Docker] to install it. NOTE: -* If you're not running as root, ensure Docker can be run [without superuser privileges][docker-non-root]. -* If you're using Docker Desktop, ensure version 4.34 or higher is installed, and [host networking is enabled][docker-desktop-host-networking]. +* If you're not running as root, ensure Docker can be run + [without superuser privileges][docker-non-root]. +* If you're using Docker Desktop, ensure version 4.34 or higher is installed, and + [host networking is enabled][docker-desktop-host-networking]. ### Python @@ -38,7 +40,8 @@ To check whether Python is installed on your system, run: python3 --version ``` -CLP requires Python 3.8 or higher. If Python isn't installed, or if the version isn't high enough, install or upgrade it by following the instructions for your OS. +CLP requires Python 3.8 or higher. If Python isn't installed, or if the version isn't high enough, +install or upgrade it by following the instructions for your OS. --- @@ -78,12 +81,14 @@ object. For example: The log file above contains two log events represented by two JSON objects printed one after the other. Whitespace is ignored, so the log events could also appear with no newlines and indentation. -If you're using JSON logs, download and extract the `clp-json` release from the [Releases][clp-releases] page, then process to the [clp-json](./clp-json.md) quick-start guide. +If you're using JSON logs, download and extract the `clp-json` release from the +[Releases][clp-releases] page, then process to the [clp-json](./clp-json.md) quick-start +guide. ### clp-text -The unstructured text flavor of CLP is appropriate for unstructured text logs, where each log event contains a -timestamp and may span one or more lines. +The unstructured text flavor of CLP is appropriate for unstructured text logs, where each log event +contains a timestamp and may span one or more lines. :::{note} If your logs do not contain timestamps or CLP can't automatically parse the timestamps in your logs, @@ -105,13 +110,16 @@ Caused by: java.net.UnknownHostException: i-e5d112ea The log file above contains two log events, both beginning with a timestamp. The first is a single line, while the second contains multiple lines. -If you're using unstructured text logs, download and extract the `clp-text` release from the [Releases][clp-releases] page, then process to the [clp-text](./clp-text.md) quick-start guide. +If you're using unstructured text logs, download and extract the `clp-text` release from the +[Releases][clp-releases] page, then process to the [clp-text](./clp-text.md) quick-start +guide. --- ## Using CLP -Once you have CLP set up, proceed to the quick start guide for your chosen flavor by clicking the link below. +Once you have CLP set up, proceed to the quick start guide for your chosen flavor by clicking the +link below. ::::{grid} 1 1 2 2 :gutter: 2 From 7b8bf616cefb82b7ae15a930be9b0cadf8591970 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Mon, 23 Jun 2025 14:14:55 +0000 Subject: [PATCH 27/40] Address rabbit comments --- docs/src/user-guide/guides-multi-node.md | 2 +- docs/src/user-guide/quick-start/index.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/user-guide/guides-multi-node.md b/docs/src/user-guide/guides-multi-node.md index 5aad660946..b3ea59162e 100644 --- a/docs/src/user-guide/guides-multi-node.md +++ b/docs/src/user-guide/guides-multi-node.md @@ -110,7 +110,7 @@ Where `` is the name of the component in the groups above. ## Using CLP -Check out the [quick-start guide](./quick-start/index) for your chosen flavour of CLP to learn how to compress and search your logs. +Check out the [quick-start guide](quick-start/index) for your chosen flavour of CLP to learn how to compress and search your logs. ::::{grid} 1 1 2 2 :gutter: 2 diff --git a/docs/src/user-guide/quick-start/index.md b/docs/src/user-guide/quick-start/index.md index 5ac225f2f5..52377ab3c5 100644 --- a/docs/src/user-guide/quick-start/index.md +++ b/docs/src/user-guide/quick-start/index.md @@ -82,7 +82,7 @@ The log file above contains two log events represented by two JSON objects print other. Whitespace is ignored, so the log events could also appear with no newlines and indentation. If you're using JSON logs, download and extract the `clp-json` release from the -[Releases][clp-releases] page, then process to the [clp-json](./clp-json.md) quick-start +[Releases][clp-releases] page, then proceed to the [clp-json](./clp-json.md) quick-start guide. ### clp-text @@ -111,7 +111,7 @@ The log file above contains two log events, both beginning with a timestamp. The line, while the second contains multiple lines. If you're using unstructured text logs, download and extract the `clp-text` release from the -[Releases][clp-releases] page, then process to the [clp-text](./clp-text.md) quick-start +[Releases][clp-releases] page, then proceed to the [clp-text](./clp-text.md) quick-start guide. --- From 734edd427f8b46022e28ef59a87c87ee67393134 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Mon, 23 Jun 2025 16:20:22 +0000 Subject: [PATCH 28/40] Fix before/after explanation for statements --- docs/src/user-guide/quick-start/clp-json.md | 2 +- docs/src/user-guide/quick-start/clp-text.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index d77f641e8b..37fb099ac7 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -102,7 +102,7 @@ To narrow your search to a specific time range: * Add `--begin-time ` to filter for log events after a certain time. * `` is the timestamp as milliseconds since the UNIX epoch. -* Add `--end-time ` to filter for log events after a certain time. +* Add `--end-time ` to filter for log events before a certain time. To perform case-insensitive searches, add the `--ignore-case` flag. diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index e2b34a5c04..fb7900c5ba 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -86,7 +86,7 @@ To narrow your search to a specific time range: * Add `--begin-time ` to filter for log events after a certain time. * `` is the timestamp as milliseconds since the UNIX epoch. -* Add `--end-time ` to filter for log events after a certain time. +* Add `--end-time ` to filter for log events before a certain time. To perform case-insensitive searches, add the `--ignore-case` flag. From 223e4545547fb39884aef4d895261a7ef9ac78e0 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Mon, 23 Jun 2025 16:25:06 +0000 Subject: [PATCH 29/40] Rabbit comments --- docs/src/user-guide/quick-start/clp-json.md | 2 +- docs/src/user-guide/quick-start/clp-text.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index 37fb099ac7..3112cb0e8a 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -3,7 +3,7 @@ This page will walk you through how to start up CLP and use it to compress and search JSON logs. :::{caution} -If you're using the `clp-json` release, you can only compress JSON logs. +If you're using the `clp-json` flavor, you can only compress JSON logs. ::: --- diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index fb7900c5ba..7b250603c9 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -4,7 +4,7 @@ This page will walk you through how to start up CLP, and then use it to compress unstructured text logs. :::{caution} -If you're using the `clp-text` release, you should only compress unstructured text logs. `clp-text` +If you're using the `clp-text` flavor, you should only compress unstructured text logs. `clp-text` is able to compress and search JSON logs as if it was unstructured text, but `clp-text` cannot query individual fields. This limitation will be addressed in a future version of CLP. ::: From a49773fe624942ba3f85923d9320e195ac27a7f8 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Mon, 30 Jun 2025 13:52:50 +0000 Subject: [PATCH 30/40] Address Kirk's comments --- docs/src/user-guide/guides-multi-node.md | 7 ++-- .../guides-using-object-storage/index.md | 6 ++-- docs/src/user-guide/index.md | 3 +- docs/src/user-guide/quick-start/clp-json.md | 22 +++++++------ .../quick-start}/clp-search-ui.png | 0 docs/src/user-guide/quick-start/clp-text.md | 10 +++--- docs/src/user-guide/quick-start/index.md | 32 +++++++++---------- 7 files changed, 42 insertions(+), 38 deletions(-) rename docs/src/{ => user-guide/quick-start}/clp-search-ui.png (100%) diff --git a/docs/src/user-guide/guides-multi-node.md b/docs/src/user-guide/guides-multi-node.md index b3ea59162e..fd9f4d8fba 100644 --- a/docs/src/user-guide/guides-multi-node.md +++ b/docs/src/user-guide/guides-multi-node.md @@ -110,21 +110,22 @@ Where `` is the name of the component in the groups above. ## Using CLP -Check out the [quick-start guide](quick-start/index) for your chosen flavour of CLP to learn how to compress and search your logs. +To learn how to compress and search your logs, check out the quick-start guide that corresponds to +the flavor of CLP you're running: ::::{grid} 1 1 2 2 :gutter: 2 :::{grid-item-card} :link: quick-start/clp-json -clp-json +Using clp-json ^^^ Compress and search JSON logs. ::: :::{grid-item-card} :link: quick-start/clp-text -clp-text +Using clp-text ^^^ Compress and search unstructured text logs. ::: diff --git a/docs/src/user-guide/guides-using-object-storage/index.md b/docs/src/user-guide/guides-using-object-storage/index.md index 92e3da90d3..428c8ef257 100644 --- a/docs/src/user-guide/guides-using-object-storage/index.md +++ b/docs/src/user-guide/guides-using-object-storage/index.md @@ -11,7 +11,7 @@ to use object storage for any combination of the three use cases (e.g., compress cache the stream files on S3, but store archives on the local filesystem). :::{note} -Currently, only the [clp-json][release-choices] flavor supports object storage. Support for +Currently, only the [clp-json][release-choices] release supports object storage. Support for `clp-text` will be added in a future release. ::: @@ -23,7 +23,7 @@ will be added in a future release. ## Prerequisites 1. This guide assumes you're able to configure, start, stop, and use a CLP cluster as described in - the [clp-json quick-start guide][release-choices]. + the [clp-json quick-start guide](../quick-start/clp-json.md). 2. An S3 bucket and [key prefix][aws-key-prefixes] containing the logs you wish to compress. 3. An S3 bucket and key prefix where you wish to store compressed archives. 4. An S3 bucket and key prefix where you wish to cache stream files. @@ -136,4 +136,4 @@ clp-usage [aws-iam-roles]: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html [aws-key-prefixes]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html [aws-sts-credentials]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html -[release-choices]: ../quick-start/clp-json +[release-choices]: ../quick-start/index.md#choosing-a-flavor diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md index 1df3d77b7e..a637c8a650 100644 --- a/docs/src/user-guide/index.md +++ b/docs/src/user-guide/index.md @@ -12,7 +12,8 @@ The sections are as follows: :link: quick-start/index Quick start ^^^ -A quick-start guide for choosing a flavour of CLP, setting it up, compressing your logs, and searching them. +A quick-start guide for choosing a flavor of CLP, setting it up, compressing your logs, and +searching them. ::: :::{grid-item-card} diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index 3112cb0e8a..7200644e12 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -1,16 +1,16 @@ # clp-json quick-start -This page will walk you through how to start up CLP and use it to compress and search JSON logs. +This page will walk you through how to start CLP and use it to compress and search JSON logs. :::{caution} -If you're using the `clp-json` flavor, you can only compress JSON logs. +If you're using a `clp-json` release, you can only compress and search JSON logs. ::: --- ## Starting CLP -To start CLP, run +To start CLP, run: ```bash sbin/start-clp.sh @@ -25,10 +25,10 @@ If CLP fails to start (e.g., due to a port conflict), try adjusting the settings ## Compressing JSON logs -To compress JSON logs, from inside the package directory, run: +To compress some JSON logs, run: ```bash -sbin/compress.sh fs --timestamp-key '' [ ...] +sbin/compress.sh --timestamp-key '' [ ...] ``` * `` is the field path of the kv-pair that contains the timestamp in each log event. @@ -43,10 +43,12 @@ sbin/compress.sh fs --timestamp-key '' [ ...] ::: * `` are paths to JSON log files or directories containing such files. - * Each JSON log file should contain each log event as a separate JSON object, - i.e., *not* as an array. + * Each JSON log file should contain each log event as a separate + [JSON object](./index.md#clp-json), i.e., *not* as an array. -Compressed logs will be stored in the `/var/bin/archives` directory. +Compressed logs will be stored in the directory specified by the `archive_output.storage.directory` +config option in `etc/clp-config.yaml` (`archive_output.storage.directory` defaults to +`var/data/archives`). :::{tip} To compress logs from object storage, see @@ -117,7 +119,7 @@ If you'd like to search your query from the web UI, CLP includes a web interface [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). -:::{image} ../../clp-search-ui.png +:::{image} clp-search-ui.png ::: The image above shows the search page after running a query. The numbered circles correspond to @@ -144,7 +146,7 @@ return more results, use the [command line](#searching-from-the-command-line). ## Stopping CLP -If you need to stop CLP, run the command +If you need to stop CLP, run: ```bash sbin/stop-clp.sh diff --git a/docs/src/clp-search-ui.png b/docs/src/user-guide/quick-start/clp-search-ui.png similarity index 100% rename from docs/src/clp-search-ui.png rename to docs/src/user-guide/quick-start/clp-search-ui.png diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index 7b250603c9..309c3c7e30 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -13,7 +13,7 @@ query individual fields. This limitation will be addressed in a future version o ## Starting CLP -To start CLP, run +To start CLP, run: ```bash sbin/start-clp.sh @@ -36,7 +36,9 @@ sbin/compress.sh [ ...] `` are paths to unstructured text log files or directories containing such files. -Compressed logs will be stored in the `/var/bin/archives` directory. +Compressed logs will be stored in the directory specified by the `archive_output.storage.directory` +config option in `etc/clp-config.yaml` (`archive_output.storage.directory` defaults to +`var/data/archives`). ### Sample logs @@ -101,7 +103,7 @@ If you'd like to search your query from the web UI, CLP includes a web interface [http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). -:::{image} ../../clp-search-ui.png +:::{image} clp-search-ui.png ::: The image above shows the search page after running a query. The numbered circles correspond to the @@ -128,7 +130,7 @@ return more results, use the [command line](#searching-from-the-command-line). ## Stopping CLP -If you need to stop CLP, run the command +If you need to stop CLP, run: ```bash sbin/stop-clp.sh diff --git a/docs/src/user-guide/quick-start/index.md b/docs/src/user-guide/quick-start/index.md index 52377ab3c5..517c068b74 100644 --- a/docs/src/user-guide/quick-start/index.md +++ b/docs/src/user-guide/quick-start/index.md @@ -1,10 +1,10 @@ # Overview -This guide describes: +This guide describes the following: -* [CLP's system requirements](#system-requirements); -* [how to choose a CLP flavor](#choosing-a-flavor); and -* [how to use CLP](#using-clp). +* [CLP's system requirements](#system-requirements) +* [How to choose a CLP flavor](#choosing-a-flavor) +* [How to use CLP](#using-clp). --- @@ -82,16 +82,15 @@ The log file above contains two log events represented by two JSON objects print other. Whitespace is ignored, so the log events could also appear with no newlines and indentation. If you're using JSON logs, download and extract the `clp-json` release from the -[Releases][clp-releases] page, then proceed to the [clp-json](./clp-json.md) quick-start -guide. +[Releases][clp-releases] page, then proceed to the [clp-json](./clp-json.md) quick-start guide. ### clp-text -The unstructured text flavor of CLP is appropriate for unstructured text logs, where each log event -contains a timestamp and may span one or more lines. +The text flavor of CLP is appropriate for unstructured text logs, where each log event contains a +timestamp and may span one or more lines. :::{note} -If your logs do not contain timestamps or CLP can't automatically parse the timestamps in your logs, +If your logs don't contain timestamps or CLP can't automatically parse the timestamps in your logs, it will treat each line as an independent log event. ::: @@ -111,15 +110,14 @@ The log file above contains two log events, both beginning with a timestamp. The line, while the second contains multiple lines. If you're using unstructured text logs, download and extract the `clp-text` release from the -[Releases][clp-releases] page, then proceed to the [clp-text](./clp-text.md) quick-start -guide. +[Releases][clp-releases] page, then proceed to the [clp-text](./clp-text.md) quick-start guide. --- ## Using CLP -Once you have CLP set up, proceed to the quick start guide for your chosen flavor by clicking the -link below. +Once you have CLP set up, proceed to the quick-start guide for your chosen flavor by clicking the +corresponding link below. ::::{grid} 1 1 2 2 :gutter: 2 @@ -128,18 +126,18 @@ link below. :link: clp-json CLP for JSON logs ^^^ -Learn about compressing and searching JSON logs. +How to compress and search JSON logs. ::: :::{grid-item-card} :link: clp-text CLP for unstructured text logs ^^^ -Learn about compressing and searching unstructured text logs. +How to compress and search unstructured text logs. ::: :::: +[clp-releases]: https://github.com/y-scope/clp/releases [Docker]: https://docs.docker.com/engine/install/ -[docker-non-root]: https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user [docker-desktop-host-networking]: https://docs.docker.com/engine/network/drivers/host/#docker-desktop -[clp-releases]: https://github.com/y-scope/clp/releases +[docker-non-root]: https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user From 7e376aaaa1bc4a0f456534656ba2b59d56431db4 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Mon, 30 Jun 2025 13:58:26 +0000 Subject: [PATCH 31/40] Correct file extension --- docs/src/user-guide/quick-start/clp-json.md | 2 +- docs/src/user-guide/quick-start/clp-text.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index 7200644e12..b0566f8e5c 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -47,7 +47,7 @@ sbin/compress.sh --timestamp-key '' [ ...] [JSON object](./index.md#clp-json), i.e., *not* as an array. Compressed logs will be stored in the directory specified by the `archive_output.storage.directory` -config option in `etc/clp-config.yaml` (`archive_output.storage.directory` defaults to +config option in `etc/clp-config.yml` (`archive_output.storage.directory` defaults to `var/data/archives`). :::{tip} diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index 309c3c7e30..3accb19b9e 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -37,7 +37,7 @@ sbin/compress.sh [ ...] `` are paths to unstructured text log files or directories containing such files. Compressed logs will be stored in the directory specified by the `archive_output.storage.directory` -config option in `etc/clp-config.yaml` (`archive_output.storage.directory` defaults to +config option in `etc/clp-config.yml` (`archive_output.storage.directory` defaults to `var/data/archives`). ### Sample logs From 31ee23b111ad86fd9174b7e521d2503efcd613e2 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Wed, 2 Jul 2025 17:54:17 +0000 Subject: [PATCH 32/40] Address Kirk's comments --- docs/src/user-guide/guides-multi-node.md | 4 +- .../guides-using-object-storage/index.md | 8 +- docs/src/user-guide/quick-start/clp-json.md | 123 ++++++++++-------- docs/src/user-guide/quick-start/clp-text.md | 97 ++++++++------ docs/src/user-guide/quick-start/index.md | 10 +- 5 files changed, 137 insertions(+), 105 deletions(-) diff --git a/docs/src/user-guide/guides-multi-node.md b/docs/src/user-guide/guides-multi-node.md index fd9f4d8fba..96c89f2ad5 100644 --- a/docs/src/user-guide/guides-multi-node.md +++ b/docs/src/user-guide/guides-multi-node.md @@ -120,14 +120,14 @@ the flavor of CLP you're running: :link: quick-start/clp-json Using clp-json ^^^ -Compress and search JSON logs. +How to compress and search JSON logs. ::: :::{grid-item-card} :link: quick-start/clp-text Using clp-text ^^^ -Compress and search unstructured text logs. +How to compress and search unstructured text logs. ::: :::: diff --git a/docs/src/user-guide/guides-using-object-storage/index.md b/docs/src/user-guide/guides-using-object-storage/index.md index 428c8ef257..0912157cf6 100644 --- a/docs/src/user-guide/guides-using-object-storage/index.md +++ b/docs/src/user-guide/guides-using-object-storage/index.md @@ -11,8 +11,8 @@ to use object storage for any combination of the three use cases (e.g., compress cache the stream files on S3, but store archives on the local filesystem). :::{note} -Currently, only the [clp-json][release-choices] release supports object storage. Support for -`clp-text` will be added in a future release. +Currently, only [clp-json][release-choices] supports object storage. Support for `clp-text` will be +added in a future release. ::: :::{note} @@ -22,8 +22,8 @@ will be added in a future release. ## Prerequisites -1. This guide assumes you're able to configure, start, stop, and use a CLP cluster as described in - the [clp-json quick-start guide](../quick-start/clp-json.md). +1. This guide assumes you're able to configure, start, stop, and use CLP as described in the + [clp-json quick-start guide](../quick-start/clp-json.md). 2. An S3 bucket and [key prefix][aws-key-prefixes] containing the logs you wish to compress. 3. An S3 bucket and key prefix where you wish to store compressed archives. 4. An S3 bucket and key prefix where you wish to cache stream files. diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index b0566f8e5c..761d7b9116 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -3,7 +3,8 @@ This page will walk you through how to start CLP and use it to compress and search JSON logs. :::{caution} -If you're using a `clp-json` release, you can only compress and search JSON logs. +If you're using a `clp-json` release, you can only compress and search JSON logs. This limitation +will be addressed in a future version of CLP. ::: --- @@ -43,8 +44,11 @@ sbin/compress.sh --timestamp-key '' [ ...] ::: * `` are paths to JSON log files or directories containing such files. - * Each JSON log file should contain each log event as a separate - [JSON object](./index.md#clp-json), i.e., *not* as an array. + * Each JSON log file should contain each log event as a + [separate JSON object](./index.md#clp-json), i.e., *not* as an array. + +The compression script will output the compression ratio of each dataset you compress, or you can +use the UI to view overall statistics. Compressed logs will be stored in the directory specified by the `archive_output.storage.directory` config option in `etc/clp-config.yml` (`archive_output.storage.directory` defaults to @@ -59,65 +63,62 @@ To compress logs from object storage, see For some sample logs, check out the open-source [datasets](../resources-datasets). -### Examining compression statistics - -The compression script used above will output the compression ratio of each dataset you compress, -or you can use the UI to view overall statistics. - --- ## Searching JSON logs -You can search through your logs using queries from the UI or from the command line. - -### Queries - -Regardless of what method you use to search, you'll need a query to find the logs you're looking -for. Queries for JSON logs take the general form of - -```bash -key: value -``` - -where `key` is the name of the JSON key you'd like to search within, and `value` is a sequence of -characters you're looking for within that key. Multiple key-value pairs can be chained together -with `AND`, `OR`, or `NOT`, like so: - -```bash -key1: value1 AND key2: value2 OR key3: value3 ... +You can search your compressed logs from CLP's [UI](#searching-from-the-ui) or the +[command line](#searching-from-the-command-line). + +Queries must be in the form of a set of conditions (predicates) on key-value pairs (kv-pairs). For +example, consider the logs in [Figure 1](#figure-1) and the query in [Figure 2](#figure-2). + +(figure-1)= +:::{card} + +```json lines +{ + "t": { + "$date": "2023-03-21T23:46:37.392" + }, + "ctx": "conn11", + "msg": "Waiting for write concern." +} +{ + "t": { + "$date": "2023-03-21T23:46:37.392" + }, + "msg": "Set last op to system time" +} ``` -There are a number of other JSON-specific syntax rules that you can use to make your searches more -powerful and effective. You can read about these rules on the -[JSON syntax reference page](../reference-json-search-syntax). - -### Searching from the command line ++++ +**Figure 1**: A set of JSON log events. +::: -If you'd like to search your query from the command line, run the following command from inside the -package: +(figure-2)= +:::{card} -```bash -sbin/search.sh '' +```sql +ctx: "conn11" AND msg: "*write concern*" ``` -To narrow your search to a specific time range: - -* Add `--begin-time ` to filter for log events after a certain time. - * `` is the timestamp as milliseconds since the UNIX epoch. -* Add `--end-time ` to filter for log events before a certain time. ++++ +**Figure 2**: An example query. +::: -To perform case-insensitive searches, add the `--ignore-case` flag. +The query in [Figure 2](#figure-2) will match log events that contain the kv-pair `"ctx": "conn11"` +as well as a kv-pair with key `"msg"` and a value that matches the wildcard query +`"*write concern*"`. This query will match the first log event in [Figure 1](#figure-1). -:::{caution} -To match the convention of other tools, by default, searches are case-**insensitive** in the UI and -searches are case-**sensitive** on the command line. -::: +A complete reference for clp-json's query syntax is available on the +[JSON syntax reference page](../reference-json-search-syntax). ### Searching from the UI -If you'd like to search your query from the web UI, CLP includes a web interface available at -[http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or -`webui.port` in `etc/clp-config.yml`, use the new values). +To search your compressed logs from CLP's UI, open [http://localhost:4000](http://localhost:4000) in +your browser (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new +values). :::{image} clp-search-ui.png ::: @@ -142,6 +143,27 @@ By default, the UI will only return 1,000 of the latest search results. To perfo return more results, use the [command line](#searching-from-the-command-line). ::: +### Searching from the command line + +To search your compressed logs from the command line, run: + +```bash +sbin/search.sh '' +``` + +To narrow your search to a specific time range: + +* Add `--begin-time ` to filter for log events after a certain time. + * `` is the timestamp as milliseconds since the UNIX epoch. +* Add `--end-time ` to filter for log events before a certain time. + +To perform case-insensitive searches, add the `--ignore-case` flag. + +:::{caution} +To match the convention of other tools, by default, searches are case-**insensitive** in the UI and +searches are case-**sensitive** on the command line. +::: + --- ## Stopping CLP @@ -151,10 +173,3 @@ If you need to stop CLP, run: ```bash sbin/stop-clp.sh ``` - ---- - -## More information - -You've reached the end of the clp-json quick-start guide. For more information on clp-json, -visit the [CLP for JSON logs](../core-clp-s) page. diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index 3accb19b9e..d2b7154f2e 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -36,72 +36,67 @@ sbin/compress.sh [ ...] `` are paths to unstructured text log files or directories containing such files. +The compression script will output the compression ratio of each dataset you compress, or you can +use the UI to view overall statistics. + Compressed logs will be stored in the directory specified by the `archive_output.storage.directory` config option in `etc/clp-config.yml` (`archive_output.storage.directory` defaults to `var/data/archives`). ### Sample logs -For some sample logs, check out the open-source [datasets](../resources-datasets). - -### Examining compression statistics - -The compression script used above will output the compression ratio of each dataset you compress, or -you can use the UI to view overall statistics. +For some sample logs, check out the [open-source datasets](../resources-datasets). --- ## Searching unstructured text logs -You can search through your logs using queries from the UI or from the command line. +You can search your compressed logs from CLP's [UI](#searching-from-the-ui) or the +[command line](#searching-from-the-command-line). -### Queries +All unstructured text log queries are written as plain text. You can use a couple of special +characters to make these queries more versatile. `*` can be used as a placeholder for an unknown +number of characters, and `?` can be used for a single character. For example, consider the logs in +[Figure 1](#figure-1) and the query in [Figure 2](#figure-2). -Regardless of what method you use to search, you'll need a query to find the logs you're looking -for. All unstructured text log queries are written as plain text. +(figure-1)= +:::{card} -You can use a couple of special characters to make these queries more versatile. `*` can be used as -a placeholder for an unknown number of characters, and `?` can be used for a single character. -For example, the query - -```bash -a*b?c +```text +1 abc +2 axbc +3 abxc +4 axxbxc +5 a b c ``` -would return all logs that contain the character `"a"` followed by any number (including zero) of -other characters, followed by `"b"`, followed by one other character, followed by `"c"`. - -There are a number of other syntax rules specific to unstructured text queries that you can use to -make your searches more powerful and effective. You can read about these rules on the -[text syntax reference page](../reference-text-search-syntax). - -### Searching from the command line ++++ +**Figure 1**: A set of unstructured text log events. +::: -If you'd like to search your query from the command line, run the following command from inside the -package: +(figure-2)= +:::{card} ```bash -sbin/search.sh '' +"a*b?c" ``` -To narrow your search to a specific time range: ++++ +**Figure 2**: An example query. +::: -* Add `--begin-time ` to filter for log events after a certain time. - * `` is the timestamp as milliseconds since the UNIX epoch. -* Add `--end-time ` to filter for log events before a certain time. +The query in [Figure 2](#figure-2) will match with lines 3, 4, and 5 in [Figure 1](#figure-1), as they are the only lines which contain the character +`"a"` followed by any number (including zero) of other characters, followed by `"b"`, followed +by one other character, followed by `"c"`. -To perform case-insensitive searches, add the `--ignore-case` flag. - -:::{caution} -To match the convention of other tools, by default, searches are case-**insensitive** in the UI and -searches are case-**sensitive** on the command line. -::: +A complete reference for `clp-text`'s query syntax is available on the +[text syntax reference page](../reference-text-search-syntax). ### Searching from the UI -If you'd like to search your query from the web UI, CLP includes a web interface available at -[http://localhost:4000](http://localhost:4000) by default (if you changed `webui.host` or -`webui.port` in `etc/clp-config.yml`, use the new values). +To search your compressed logs from CLP's UI, open [http://localhost:4000](http://localhost:4000) in +your browser (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new +values). :::{image} clp-search-ui.png ::: @@ -126,6 +121,28 @@ By default, the UI will only return 1,000 of the latest search results. To perfo return more results, use the [command line](#searching-from-the-command-line). ::: +### Searching from the command line + +To search your compressed logs from the command line, run: + +```bash +sbin/search.sh '' +``` + +To narrow your search to a specific time range: + +* Add `--begin-time ` to filter for log events after a certain time. + * `` is the timestamp as milliseconds since the UNIX epoch. +* Add `--end-time ` to filter for log events before a certain time. + +To perform case-insensitive searches, add the `--ignore-case` flag. + +:::{caution} +To match the convention of other tools, by default, searches are case-**insensitive** in the UI and +searches are case-**sensitive** on the command line. +::: + + --- ## Stopping CLP diff --git a/docs/src/user-guide/quick-start/index.md b/docs/src/user-guide/quick-start/index.md index 517c068b74..029dde79d8 100644 --- a/docs/src/user-guide/quick-start/index.md +++ b/docs/src/user-guide/quick-start/index.md @@ -23,7 +23,7 @@ To check whether Docker is installed on your system, run: docker version ``` -If Docker isn't installed, follow these [instructions][Docker] to install it. +If Docker isn't installed, follow [these instructions][Docker] to install it. NOTE: @@ -82,7 +82,7 @@ The log file above contains two log events represented by two JSON objects print other. Whitespace is ignored, so the log events could also appear with no newlines and indentation. If you're using JSON logs, download and extract the `clp-json` release from the -[Releases][clp-releases] page, then proceed to the [clp-json](./clp-json.md) quick-start guide. +[Releases][clp-releases] page, then proceed to the [clp-json quick-start](./clp-json.md) guide. ### clp-text @@ -110,14 +110,14 @@ The log file above contains two log events, both beginning with a timestamp. The line, while the second contains multiple lines. If you're using unstructured text logs, download and extract the `clp-text` release from the -[Releases][clp-releases] page, then proceed to the [clp-text](./clp-text.md) quick-start guide. +[Releases][clp-releases] page, then proceed to the [clp-text quick-start](./clp-text.md) guide. --- ## Using CLP -Once you have CLP set up, proceed to the quick-start guide for your chosen flavor by clicking the -corresponding link below. +Once you've installed CLP's requirements and downloaded a CLP release, proceed to the quick-start +guide for your chosen flavor by clicking the corresponding link below. ::::{grid} 1 1 2 2 :gutter: 2 From e302d38a3ec56d3a133a24b08f9d817a7fd7624f Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Wed, 2 Jul 2025 18:01:07 +0000 Subject: [PATCH 33/40] Rabbit/lint --- docs/src/user-guide/quick-start/clp-json.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index 761d7b9116..3e480b7453 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -44,7 +44,7 @@ sbin/compress.sh --timestamp-key '' [ ...] ::: * `` are paths to JSON log files or directories containing such files. - * Each JSON log file should contain each log event as a + * Each JSON log file should contain each log event as a [separate JSON object](./index.md#clp-json), i.e., *not* as an array. The compression script will output the compression ratio of each dataset you compress, or you can From f3a538ee43be47adfe428478aa720733e3335574 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 3 Jul 2025 15:06:03 +0000 Subject: [PATCH 34/40] Make changes to clp-text.md to mirror changes in clp-json.md --- docs/src/user-guide/quick-start/clp-text.md | 29 ++++++++------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index d2b7154f2e..832dda12cd 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -1,10 +1,10 @@ # clp-text quick-start -This page will walk you through how to start up CLP, and then use it to compress and search -unstructured text logs. +This page will walk you through how to start CLP and use it to compress and search unstructured +text logs. :::{caution} -If you're using the `clp-text` flavor, you should only compress unstructured text logs. `clp-text` +If you're using a `clp-text` release, you should only compress unstructured text logs. `clp-text` is able to compress and search JSON logs as if it was unstructured text, but `clp-text` cannot query individual fields. This limitation will be addressed in a future version of CLP. ::: @@ -28,7 +28,7 @@ If CLP fails to start (e.g., due to a port conflict), try adjusting the settings ## Compressing unstructured text logs -To compress unstructured text logs, run the following from inside the package directory: +To compress some unstructured text logs, run: ```bash sbin/compress.sh [ ...] @@ -54,7 +54,7 @@ For some sample logs, check out the [open-source datasets](../resources-datasets You can search your compressed logs from CLP's [UI](#searching-from-the-ui) or the [command line](#searching-from-the-command-line). -All unstructured text log queries are written as plain text. You can use a couple of special +Unstructured text log queries are written as plain text. You can use a couple of special characters to make these queries more versatile. `*` can be used as a placeholder for an unknown number of characters, and `?` can be used for a single character. For example, consider the logs in [Figure 1](#figure-1) and the query in [Figure 2](#figure-2). @@ -85,11 +85,11 @@ number of characters, and `?` can be used for a single character. For example, c **Figure 2**: An example query. ::: -The query in [Figure 2](#figure-2) will match with lines 3, 4, and 5 in [Figure 1](#figure-1), as they are the only lines which contain the character -`"a"` followed by any number (including zero) of other characters, followed by `"b"`, followed -by one other character, followed by `"c"`. +The query in [Figure 2](#figure-2) will match with lines 3, 4, and 5 in [Figure 1](#figure-1), as +they are the only lines which contain the character `"a"` followed by any number (including zero) +of other characters, followed by `"b"`, followed by one other character, followed by `"c"`. -A complete reference for `clp-text`'s query syntax is available on the +A complete reference for clp-text's query syntax is available on the [text syntax reference page](../reference-text-search-syntax). ### Searching from the UI @@ -101,8 +101,8 @@ values). :::{image} clp-search-ui.png ::: -The image above shows the search page after running a query. The numbered circles correspond to the -following features: +The image above shows the search page after running a query. The numbered circles correspond to +the following features: 1. The search box is where you can enter your query. 2. The timeline shows the number of results across the time range of your query. @@ -152,10 +152,3 @@ If you need to stop CLP, run: ```bash sbin/stop-clp.sh ``` - ---- - -## More information - -You've reached the end of the clp-text quick-start guide. For more information on clp-text, -visit the [CLP for unstructured text logs](../core-unstructured/index) page. From 61eedc360760e69d5b80fc372669d803650e642c Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 3 Jul 2025 15:08:13 +0000 Subject: [PATCH 35/40] Lint changes --- docs/src/user-guide/quick-start/clp-json.md | 4 ++-- docs/src/user-guide/quick-start/clp-text.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index 3e480b7453..954d0ac64b 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -131,8 +131,8 @@ the following features: * You can click and drag to zoom into a time range, or use the time range filter in (4). 3. The table displays the search results for your query. 4. Clicking the icon reveals additional filters for your query. - * The time range filter allows you to specify the period of time that matching log events must be - in. + * The time range filter allows you to specify the period of time that matching log events must + be in. * The case sensitivity filter allows you to specify whether CLP should respect the case of your query. 5. Clicking the icon reveals options for displaying results. diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index 832dda12cd..dcaa62ec79 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -94,8 +94,8 @@ A complete reference for clp-text's query syntax is available on the ### Searching from the UI -To search your compressed logs from CLP's UI, open [http://localhost:4000](http://localhost:4000) in -your browser (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new +To search your compressed logs from CLP's UI, open [http://localhost:4000](http://localhost:4000) +in your browser (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). :::{image} clp-search-ui.png @@ -109,8 +109,8 @@ the following features: * You can click and drag to zoom into a time range, or use the time range filter in (4). 3. The table displays the search results for your query. 4. Clicking the icon reveals additional filters for your query. - * The time range filter allows you to specify the period of time that matching log events must be - in. + * The time range filter allows you to specify the period of time that matching log events must + be in. * The case sensitivity filter allows you to specify whether CLP should respect the case of your query. 5. Clicking the icon reveals options for displaying results. From 5fed1f26c9a563a2ce24b5f51c0a83cdcef4b7d5 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 3 Jul 2025 15:17:43 +0000 Subject: [PATCH 36/40] Rabbit --- docs/src/user-guide/quick-start/clp-text.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index dcaa62ec79..df2ecc5230 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -142,7 +142,6 @@ To match the convention of other tools, by default, searches are case-**insensit searches are case-**sensitive** on the command line. ::: - --- ## Stopping CLP From a10aa547b20b413249936cfb33d20e1f9252096e Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 3 Jul 2025 18:30:50 +0000 Subject: [PATCH 37/40] Address Kirk's comments --- docs/src/user-guide/quick-start/clp-json.md | 37 +++++++------- docs/src/user-guide/quick-start/clp-text.md | 53 +++++++++++---------- 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index 954d0ac64b..7105dea173 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -61,7 +61,7 @@ To compress logs from object storage, see ### Sample logs -For some sample logs, check out the open-source [datasets](../resources-datasets). +For some sample logs, check out the [open-source datasets](../resources-datasets). --- @@ -70,12 +70,24 @@ For some sample logs, check out the open-source [datasets](../resources-datasets You can search your compressed logs from CLP's [UI](#searching-from-the-ui) or the [command line](#searching-from-the-command-line). -Queries must be in the form of a set of conditions (predicates) on key-value pairs (kv-pairs). For -example, consider the logs in [Figure 1](#figure-1) and the query in [Figure 2](#figure-2). +In clp-json, queries are written as a set of conditions (predicates) on key-value pairs (kv-pairs). +For example, [Figure 1](#figure-1) shows a query that matches the first log event in +[Figure 2](#figure-2). (figure-1)= :::{card} +```sql +ctx: "conn11" AND msg: "*write concern*" +``` + ++++ +**Figure 1**: An example query. +::: + +(figure-2)= +:::{card} + ```json lines { "t": { @@ -93,23 +105,12 @@ example, consider the logs in [Figure 1](#figure-1) and the query in [Figure 2]( ``` +++ -**Figure 1**: A set of JSON log events. -::: - -(figure-2)= -:::{card} - -```sql -ctx: "conn11" AND msg: "*write concern*" -``` - -+++ -**Figure 2**: An example query. +**Figure 2**: A set of JSON log events. ::: The query in [Figure 2](#figure-2) will match log events that contain the kv-pair `"ctx": "conn11"` as well as a kv-pair with key `"msg"` and a value that matches the wildcard query -`"*write concern*"`. This query will match the first log event in [Figure 1](#figure-1). +`"*write concern*"`. A complete reference for clp-json's query syntax is available on the [JSON syntax reference page](../reference-json-search-syntax). @@ -131,8 +132,8 @@ the following features: * You can click and drag to zoom into a time range, or use the time range filter in (4). 3. The table displays the search results for your query. 4. Clicking the icon reveals additional filters for your query. - * The time range filter allows you to specify the period of time that matching log events must - be in. + * The time range filter allows you to specify the period of time that matching log events must be + in. * The case sensitivity filter allows you to specify whether CLP should respect the case of your query. 5. Clicking the icon reveals options for displaying results. diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index df2ecc5230..5f6811f3b1 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -5,7 +5,7 @@ text logs. :::{caution} If you're using a `clp-text` release, you should only compress unstructured text logs. `clp-text` -is able to compress and search JSON logs as if it was unstructured text, but `clp-text` cannot +is able to compress and search JSON logs as if they were unstructured text, but `clp-text` cannot query individual fields. This limitation will be addressed in a future version of CLP. ::: @@ -54,63 +54,68 @@ For some sample logs, check out the [open-source datasets](../resources-datasets You can search your compressed logs from CLP's [UI](#searching-from-the-ui) or the [command line](#searching-from-the-command-line). -Unstructured text log queries are written as plain text. You can use a couple of special -characters to make these queries more versatile. `*` can be used as a placeholder for an unknown -number of characters, and `?` can be used for a single character. For example, consider the logs in -[Figure 1](#figure-1) and the query in [Figure 2](#figure-2). +In clp-text, queries are written as wildcard expressions. A wildcard expression is a plain text +query where: + +* `*` matches zero or more characters +* `?` matches any single character + +For example, consider the query in [Figure 1](#figure-1) and the logs in [Figure 2](#figure-2). (figure-1)= :::{card} -```text -1 abc -2 axbc -3 abxc -4 axxbxc -5 a b c +```bash +"INFO container_? Transitioned*ACQUIRED" ``` +++ -**Figure 1**: A set of unstructured text log events. +**Figure 1**: An example query. ::: (figure-2)= :::{card} -```bash -"a*b?c" +```text +2015-03-23T15:50:17.926Z INFO container_1 Transitioned from ALLOCATED to ACQUIRED +2015-03-23T15:50:17.927Z ERROR Scheduler: Error trying to assign container token +java.lang.IllegalArgumentException: java.net.UnknownHostException: i-e5d112ea + at org.apache.hadoop.security.buildTokenService(SecurityUtil.java:374) + at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2033) +Caused by: java.net.UnknownHostException: i-e5d112ea + ... 17 more ``` +++ -**Figure 2**: An example query. +**Figure 2**: A set of unstructured text log events. ::: -The query in [Figure 2](#figure-2) will match with lines 3, 4, and 5 in [Figure 1](#figure-1), as -they are the only lines which contain the character `"a"` followed by any number (including zero) -of other characters, followed by `"b"`, followed by one other character, followed by `"c"`. +The query in [Figure 2](#figure-2) will match with the first log message, as the `?` will stand in +for any one character (in this case, `1`), and the `*` will stand in for the characters +` from ALLOCATED to `. A complete reference for clp-text's query syntax is available on the [text syntax reference page](../reference-text-search-syntax). ### Searching from the UI -To search your compressed logs from CLP's UI, open [http://localhost:4000](http://localhost:4000) -in your browser (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new +To search your compressed logs from CLP's UI, open [http://localhost:4000](http://localhost:4000) in +your browser (if you changed `webui.host` or `webui.port` in `etc/clp-config.yml`, use the new values). :::{image} clp-search-ui.png ::: -The image above shows the search page after running a query. The numbered circles correspond to -the following features: +The image above shows the search page after running a query. The numbered circles correspond to the +following features: 1. The search box is where you can enter your query. 2. The timeline shows the number of results across the time range of your query. * You can click and drag to zoom into a time range, or use the time range filter in (4). 3. The table displays the search results for your query. 4. Clicking the icon reveals additional filters for your query. - * The time range filter allows you to specify the period of time that matching log events must - be in. + * The time range filter allows you to specify the period of time that matching log events must be + in. * The case sensitivity filter allows you to specify whether CLP should respect the case of your query. 5. Clicking the icon reveals options for displaying results. From 6d6c779226b589f733a01c4c2062820c04e654f7 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 3 Jul 2025 19:46:55 +0000 Subject: [PATCH 38/40] Address Kirk's comments --- docs/src/user-guide/quick-start/clp-json.md | 2 +- docs/src/user-guide/quick-start/clp-text.md | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index 7105dea173..8e258052c7 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -113,7 +113,7 @@ as well as a kv-pair with key `"msg"` and a value that matches the wildcard quer `"*write concern*"`. A complete reference for clp-json's query syntax is available on the -[JSON syntax reference page](../reference-json-search-syntax). +[syntax reference page](../reference-json-search-syntax). ### Searching from the UI diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index 5f6811f3b1..29d2054140 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -90,12 +90,11 @@ Caused by: java.net.UnknownHostException: i-e5d112ea **Figure 2**: A set of unstructured text log events. ::: -The query in [Figure 2](#figure-2) will match with the first log message, as the `?` will stand in -for any one character (in this case, `1`), and the `*` will stand in for the characters -` from ALLOCATED to `. +The query in [Figure 2](#figure-2) will match with the first log message, as the `?` will match the +character "1", and the `*` match the text " from ALLOCATED to ". A complete reference for clp-text's query syntax is available on the -[text syntax reference page](../reference-text-search-syntax). +[syntax reference page](../reference-text-search-syntax). ### Searching from the UI From 0db424f591cf6be82916b934c9590fee6ea0663a Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 3 Jul 2025 19:53:07 +0000 Subject: [PATCH 39/40] Rabbit --- docs/src/user-guide/quick-start/clp-json.md | 2 +- docs/src/user-guide/quick-start/clp-text.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/user-guide/quick-start/clp-json.md b/docs/src/user-guide/quick-start/clp-json.md index 8e258052c7..d83971fd6e 100644 --- a/docs/src/user-guide/quick-start/clp-json.md +++ b/docs/src/user-guide/quick-start/clp-json.md @@ -108,7 +108,7 @@ ctx: "conn11" AND msg: "*write concern*" **Figure 2**: A set of JSON log events. ::: -The query in [Figure 2](#figure-2) will match log events that contain the kv-pair `"ctx": "conn11"` +The query in [Figure 1](#figure-1) will match log events that contain the kv-pair `"ctx": "conn11"` as well as a kv-pair with key `"msg"` and a value that matches the wildcard query `"*write concern*"`. diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index 29d2054140..438503f717 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -90,7 +90,7 @@ Caused by: java.net.UnknownHostException: i-e5d112ea **Figure 2**: A set of unstructured text log events. ::: -The query in [Figure 2](#figure-2) will match with the first log message, as the `?` will match the +The query in [Figure 1](#figure-1) will match with the first log message, as the `?` will match the character "1", and the `*` match the text " from ALLOCATED to ". A complete reference for clp-text's query syntax is available on the From ab2181df6fa9719487e67bcf81c5e04c78933042 Mon Sep 17 00:00:00 2001 From: Quinn Mitchell Date: Thu, 3 Jul 2025 20:25:34 +0000 Subject: [PATCH 40/40] Address Kirk's comment --- docs/src/user-guide/quick-start/clp-text.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/user-guide/quick-start/clp-text.md b/docs/src/user-guide/quick-start/clp-text.md index 438503f717..cfc3469b71 100644 --- a/docs/src/user-guide/quick-start/clp-text.md +++ b/docs/src/user-guide/quick-start/clp-text.md @@ -91,7 +91,7 @@ Caused by: java.net.UnknownHostException: i-e5d112ea ::: The query in [Figure 1](#figure-1) will match with the first log message, as the `?` will match the -character "1", and the `*` match the text " from ALLOCATED to ". +character "1", and the `*` will match the text " from ALLOCATED to ". A complete reference for clp-text's query syntax is available on the [syntax reference page](../reference-text-search-syntax).