Index & deployment name customization

deployment/operations/cf/add-firehose-to-syslog-uaa-clients.yml

@@ -3,7 +3,7 @@
 
 # UAA client for firehose-to-syslog
 - type: replace
-  path: /instance_groups/name=uaa/jobs/name=uaa/properties/uaa/clients/firehose-to-syslog?
+  path: /instance_groups/name=uaa/jobs/name=uaa/properties/uaa/clients/((uaa_clients_firehose_client_id))?
   value:
     override: true
     authorized-grant-types: client_credentials

deployment/operations/cloudfoundry.yml

@@ -77,9 +77,10 @@
       ingestor: {from: ingestor_link}
     properties:
       cloudfoundry:
-        firehose_client_id: firehose-to-syslog
+        firehose_client_id: "((firehose_client_id))"
         firehose_client_secret: "((firehose_client_secret))"
         skip_ssl_validation: true
+        firehose_subscription_id: "((firehose_client_id))"
 
 - type: replace
   path: /instance_groups/name=ingestor/jobs/name=syslog_forwarder/properties/syslog_forwarder/config/-
@@ -113,6 +114,7 @@
     properties:
       kibana-auth:
         cloudfoundry:
+          client_id: "((kibana_oauth2_client_id))"
           client_secret: "((kibana_oauth2_client_secret))"
           skip_ssl_validation: true
 
@@ -138,7 +140,7 @@
           port: 80
           registration_interval: 60s
           uris:
-          - "logs.((system_domain))"
+          - "((kibana_hostname)).((system_domain))"
 
 - type: replace
   path: /instance_groups/name=kibana/jobs/name=kibana/properties/kibana/plugins?

jobs/ingestor_syslog/spec

@@ -158,9 +158,27 @@ properties:
   logstash_parser.elasticsearch.document_id:
     description: "Use a specific, dynamic ID rather than an auto-generated identifier."
     default: ~
+  logstash_parser.elasticsearch.index_name.platform.base:
+    description: "The platform index base name"
+    default: "platform"
+  logstash_parser.elasticsearch.index_name.platform.period:
+    description: "The platform index period - Joda-Time format https://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html"
+    default: "YYYY.MM.dd"
+  logstash_parser.elasticsearch.index_name.app.base:
+    description: "The app index base name"
+    default: "app"
+  logstash_parser.elasticsearch.index_name.app.append_org_to_base:
+    description: "Whether to add the CF Org to the index base name"
+    default: true
+  logstash_parser.elasticsearch.index_name.app.append_space_to_base:
+    description: "Whether to add the CF Space to the index base name"
+    default: false
+  logstash_parser.elasticsearch.index_name.app.period:
+    description: "The app index period - Joda-Time format https://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html"
+    default: "YYYY.MM.dd"
   logstash_parser.elasticsearch.index:
     description: "The specific, dynamic index name to write events to."
-    default: "logstash-%{+YYYY.MM.dd}"
+    default: "logs-%{[@metadata][index_base]}-%{[@metadata][index_period]}"
   logstash_parser.elasticsearch.index_type:
     description: "The specific, dynamic index type name to write events to."
     default: "%{@type}"

jobs/ingestor_syslog/templates/config/filters_post.conf.erb

@@ -20,3 +20,33 @@
       }
     }
 
+
+    mutate {
+      add_field => { "[@metadata][index_base]" => "<%=p('logstash_parser.elasticsearch.index_name.platform.base')%>" }
+      add_field => { "[@metadata][index_period]" => "%{+<%=p('logstash_parser.elasticsearch.index_name.platform.period')%>}" } # YYYY.MM.dd
+    }
+
+    # Custom Override @metadata.index
+    if [@index_type] == "app" {
+        mutate {
+            replace => { "[@metadata][index_base]" => "<%=p('logstash_parser.elasticsearch.index_name.app.base')%>" }
+            replace => { "[@metadata][index_period]" => "%{+<%=p('logstash_parser.elasticsearch.index_name.app.period')%>}" } # YYYY.MM, xxxx.ww, YYYY.MM.dd
+        }
+        <% if p('logstash_parser.elasticsearch.index_name.app.append_org_to_base') %>
+        if [@cf][org] {
+            mutate {
+                replace => { "[@metadata][index_base]" => "%{[@metadata][index_base]}-%{[@cf][org]}" }
+            }
+        }
+        <% end %>
+        <% if p('logstash_parser.elasticsearch.index_name.app.append_space_to_base') %>
+        if [@cf][space] {
+            mutate {
+                replace => { "[@metadata][index_base]" => "%{[@metadata][index_base]}-%{[@cf][space]}" }
+            }
+        }
+        <% end %>
+        mutate {
+            lowercase => [ "[@metadata][index_base]" ]
+        }
+    }