Allow code blocks to import code from files or URLs

code_snippets/ClickStack/config-unstructured-logs-with-processor.yaml

@@ -0,0 +1,43 @@
+receivers:
+  filelog:
+    include:
+      - /opt/data/logs/access-unstructured.log
+    start_at: beginning
+    operators:
+      - type: regex_parser
+        regex: '^(?P<ip>[\d.]+)\s+-\s+-\s+\[(?P<timestamp>[^\]]+)\]\s+"(?P<method>[A-Z]+)\s+(?P<url>[^\s]+)\s+HTTP/[^\s]+"\s+(?P<status>\d+)\s+(?P<size>\d+)\s+"(?P<referrer>[^"]*)"\s+"(?P<user_agent>[^"]*)"'
+        timestamp:
+          parse_from: attributes.timestamp
+          layout: '%d/%b/%Y:%H:%M:%S %z'
+          #22/Jan/2019:03:56:14 +0330
+processors:
+  batch:
+    timeout: 1s
+    send_batch_size: 100
+  memory_limiter:
+    check_interval: 1s
+    limit_mib: 2048
+    spike_limit_mib: 256
+exporters:
+  # HTTP setup
+  otlphttp/hdx:
+    endpoint: 'http://localhost:4318'
+    headers:
+      authorization: <YOUR_INGESTION_API_KEY>
+    compression: gzip
+
+  # gRPC setup (alternative)
+  otlp/hdx:
+    endpoint: 'localhost:4317'
+    headers:
+      authorization: <YOUR_API_INGESTION_KEY>
+    compression: gzip
+service:
+  telemetry:
+    metrics:
+      address: 0.0.0.0:9888 # Modified as 2 collectors running on same host
+  pipelines:
+    logs:
+      receivers: [filelog]
+      processors: [batch]
+      exporters: [otlphttp/hdx]

docs/use-cases/observability/clickstack/ingesting-data/collector.md

@@ -161,51 +161,8 @@ The following configuration shows collection of this [unstructured log file](htt
 
 Note the use of operators to extract structure from the log lines (`regex_parser`) and filter events, along with a processor to batch events and limit memory usage.
 
-```yaml
-# config-unstructured-logs-with-processor.yaml
-receivers:
-  filelog:
-    include:
-      - /opt/data/logs/access-unstructured.log
-    start_at: beginning
-    operators:
-      - type: regex_parser
-        regex: '^(?P<ip>[\d.]+)\s+-\s+-\s+\[(?P<timestamp>[^\]]+)\]\s+"(?P<method>[A-Z]+)\s+(?P<url>[^\s]+)\s+HTTP/[^\s]+"\s+(?P<status>\d+)\s+(?P<size>\d+)\s+"(?P<referrer>[^"]*)"\s+"(?P<user_agent>[^"]*)"'
-        timestamp:
-          parse_from: attributes.timestamp
-          layout: '%d/%b/%Y:%H:%M:%S %z'
-          #22/Jan/2019:03:56:14 +0330
-processors:
-  batch:
-    timeout: 1s
-    send_batch_size: 100
-  memory_limiter:
-    check_interval: 1s
-    limit_mib: 2048
-    spike_limit_mib: 256
-exporters:
-  # HTTP setup
-  otlphttp/hdx:
-    endpoint: 'http://localhost:4318'
-    headers:
-      authorization: <YOUR_INGESTION_API_KEY>
-    compression: gzip
-
-  # gRPC setup (alternative)
-  otlp/hdx:
-    endpoint: 'localhost:4317'
-    headers:
-      authorization: <YOUR_API_INGESTION_KEY>
-    compression: gzip
-service:
-  telemetry:
-    metrics:
-      address: 0.0.0.0:9888 # Modified as 2 collectors running on same host
-  pipelines:
-    logs:
-      receivers: [filelog]
-      processors: [batch]
-      exporters: [otlphttp/hdx]
+```yaml file=code_snippets/ClickStack/config-unstructured-logs-with-processor.yaml
+place holder text
 ```
 
 Note the need to include an [authorization header containing your ingestion API key](#securing-the-collector) in any OTLP communication.

docusaurus.config.en.js

@@ -5,6 +5,7 @@ import chHeader from "./plugins/header.js";
 import fixLinks from "./src/hooks/fixLinks.js";
 const path = require('path');
 const remarkCustomBlocks = require('./plugins/remark-custom-blocks');
+const remarkCodeImport = require('./plugins/remark-code-import');
 
 // Import custom plugins
 const { customParseFrontMatter } = require('./plugins/frontmatter-validation/customParseFrontMatter');
@@ -152,7 +153,7 @@ const config = {
           showLastUpdateTime: false,
           sidebarCollapsed: true,
           routeBasePath: "/",
-          remarkPlugins: [math, remarkCustomBlocks, glossaryTransformer],
+          remarkPlugins: [math, remarkCustomBlocks, glossaryTransformer, [remarkCodeImport, { baseDir: __dirname }]],
           beforeDefaultRemarkPlugins: [fixLinks],
           rehypePlugins: [katex],
         },

package.json

@@ -63,6 +63,7 @@
     "node-fetch": "^3.3.2",
     "numeral": "^2.0.6",
     "prism-react-renderer": "^2.4.1",
+    "raw-loader": "^4.0.2",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-medium-image-zoom": "^5.2.14",

plugins/remark-code-import.js

@@ -0,0 +1,47 @@
+const fs = require('fs');
+const path = require('path');
+const { visit } = require('unist-util-visit');
+
+function createCodeImportPlugin(options = {}) {
+  const { baseDir = process.cwd() } = options;
+  return function transformer(tree, file) {
+    visit(tree, 'code', (node) => {
+      const { lang, meta } = node;
+
+      // Check if file= is in meta or lang
+      let fileMatch = null;
+      let filePath = null;
+
+      if (meta) {
+        fileMatch = meta.match(/file=([^\s]+)/);
+      }
+      if (!fileMatch && lang) {
+        fileMatch = lang.match(/file=([^\s]+)/);
+        if (fileMatch) {
+          // Extract real language (everything before file=)
+          const realLang = lang.split(/\s+file=/)[0].trim();
+          node.lang = realLang;
+        }
+      }
+
+      if (!fileMatch) return;
+
+      filePath = fileMatch[1];
+      const absolutePath = path.resolve(baseDir, filePath);
+
+      try {
+        const content = fs.readFileSync(absolutePath, 'utf8');
+        node.value = content;
+
+        // Clean up meta if it had file=
+        if (meta && meta.includes('file=')) {
+          node.meta = meta.replace(/file=[^\s]+\s*/, '').trim() || null;
+        }
+      } catch (error) {
+        console.warn(`Could not read file ${filePath}: ${error.message}`);
+      }
+    });
+  };
+};
+
+module.exports = createCodeImportPlugin;

yarn.lock

@@ -11754,6 +11754,14 @@ [email protected]:
     iconv-lite "0.4.24"
     unpipe "1.0.0"
 
+raw-loader@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/raw-loader/-/raw-loader-4.0.2.tgz#1aac6b7d1ad1501e66efdac1522c73e59a584eb6"
+  integrity sha512-ZnScIV3ag9A4wPX/ZayxL/jZH+euYb6FcUinPcgiQW0+UBtEv0O6Q3lGd3cqJ+GHH+rksEv3Pj99oxJ3u3VIKA==
+  dependencies:
+    loader-utils "^2.0.0"
+    schema-utils "^3.0.0"
+
 [email protected], rc@^1.2.7:
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed"