Version 0.2.0 (#7)

tamirkamara · web-flow · commit e76686e9cb2d · 2019-01-16T14:41:36.000+02:00
* Include Kusto SDK 1.0.0-BETA-01, include some tests, fix password bug
* change logstash branch from 6.0 to 6.5 to make ci pass
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,20 @@
+sudo: false
+language: ruby
+cache: bundler
+matrix:
+  include:
+  - rvm: jruby-9.1.13.0
+    env: LOGSTASH_BRANCH=master
+  - rvm: jruby-9.1.13.0
+    env: LOGSTASH_BRANCH=6.x
+  - rvm: jruby-9.1.13.0
+    env: LOGSTASH_BRANCH=6.5
+  fast_finish: true
+install: true
+script: ci/build.sh
+jdk: oraclejdk8
+deploy:
+  provider: rubygems
+  api_key: $rubygems_api_key
+  on:
+    tags: true
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -11,6 +11,15 @@
                 "kind": "build",
                 "isDefault": true
             }
+        },
+        {
+            "label": "Run tests",
+            "type": "shell",
+            "command": "bundle exec rspec spec",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
         }
     ]
 }
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,3 +2,7 @@
   - Plugin created with the logstash plugin generator
 ## 0.1.6
   - plugin published to the public. supports ingestion json events into a specific table-database (without dynamic routing currently)
+## 0.1.7
+  - fixed app_key (password) bug, include 0.1.7 of the kusto-java-sdk to allow working through a proxy
+## 0.2.0
+  - move to version 1.0.0-BETA-01 of azure-kusto-kava sdk
diff --git a/README.md b/README.md
@@ -1,6 +1,7 @@
 # Logstash Output Plugin for Azure Data Explorer (Kusto)
 
-[![Travis Build Status](https://travis-ci.org/Azure/logstash-output-kusto.svg)](https://travis-ci.org/Azure/logstash-output-kusto)
+master: [![Build Status](https://travis-ci.org/Azure/logstash-output-kusto.svg)](https://travis-ci.org/Azure/logstash-output-kusto) 
+dev: [![Build Status](https://travis-ci.org/Azure/logstash-output-kusto.svg?branch=dev)](https://travis-ci.org/Azure/logstash-output-kusto)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
diff --git a/ci/build.sh b/ci/build.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+# version: 1
+########################################################
+#
+# AUTOMATICALLY GENERATED! DO NOT EDIT
+#
+########################################################
+set -e
+
+echo "Starting build process in: `pwd`"
+source ./ci/setup.sh
+
+if [[ -f "ci/run.sh" ]]; then
+    echo "Running custom build script in: `pwd`/ci/run.sh"
+    source ./ci/run.sh
+else
+    echo "Running default build scripts in: `pwd`/ci/build.sh"
+    bundle install
+    bundle exec rake vendor
+    bundle exec rspec spec
+fi
diff --git a/ci/setup.sh b/ci/setup.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# version: 1
+########################################################
+#
+# AUTOMATICALLY GENERATED! DO NOT EDIT
+#
+########################################################
+set -e
+if [ "$LOGSTASH_BRANCH" ]; then
+    echo "Building plugin using Logstash source"
+    BASE_DIR=`pwd`
+    echo "Checking out branch: $LOGSTASH_BRANCH"
+    git clone -b $LOGSTASH_BRANCH https://github.com/elastic/logstash.git ../../logstash --depth 1
+    printf "Checked out Logstash revision: %s\n" "$(git -C ../../logstash rev-parse HEAD)"
+    cd ../../logstash
+    echo "Building plugins with Logstash version:"
+    cat versions.yml
+    echo "---"
+    # We need to build the jars for that specific version
+    echo "Running gradle assemble in: `pwd`"
+    ./gradlew assemble
+    cd $BASE_DIR
+    export LOGSTASH_SOURCE=1
+else
+    echo "Building plugin using released gems on rubygems"
+fi
diff --git a/lib/kusto/kusto-ingest-1.0.0-BETA-01-jar-with-dependencies.jar b/lib/kusto/kusto-ingest-1.0.0-BETA-01-jar-with-dependencies.jar
diff --git a/lib/logstash/outputs/kusto.rb b/lib/logstash/outputs/kusto.rb
@@ -8,7 +8,7 @@
 require 'logstash/outputs/kusto/interval'
 
 ##
-# This plugin send messages to Azure Kusto in batches.
+# This plugin sends messages to Azure Kusto in batches.
 #
 class LogStash::Outputs::Kusto < LogStash::Outputs::Base
   config_name 'kusto'
@@ -152,9 +152,14 @@ def register
   private
   def validate_path
     if (root_directory =~ FIELD_REF) != nil
-      @logger.error('File: The starting part of the path should not be dynamic.', path: @path)
+      @logger.error('The starting part of the path should not be dynamic.', path: @path)
       raise LogStash::ConfigurationError.new('The starting part of the path should not be dynamic.')
     end
+
+    if !path_with_field_ref?
+      @logger.error('Path should include some time related fields to allow for file rotation.', path: @path)
+      raise LogStash::ConfigurationError.new('Path should include some time related fields to allow for file rotation.')
+    end
   end
 
   private 
diff --git a/lib/logstash/outputs/kusto/ingestor.rb b/lib/logstash/outputs/kusto/ingestor.rb
@@ -9,7 +9,7 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
   # This handles the overall logic and communication with Kusto
   #
   class Ingestor
-    require 'kusto/KustoClient-0.1.6.jar'
+    require 'kusto/kusto-ingest-1.0.0-BETA-01-jar-with-dependencies.jar'
 
     RETRY_DELAY_SECONDS = 3
     DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new(
@@ -29,11 +29,11 @@ def initialize(ingest_url, app_id, app_key, app_tenant, database, table, mapping
 
       @logger.debug('Preparing Kusto resources.')
 
-      kusto_connection_string = Java::KustoConnectionStringBuilder.createWithAadApplicationCredentials(ingest_url, app_id, app_key, app_tenant)
+      kusto_connection_string = Java::com.microsoft.azure.kusto.data.ConnectionStringBuilder.createWithAadApplicationCredentials(ingest_url, app_id, app_key.value, app_tenant)
 
-      @kusto_client = Java::KustoIngestClient.new(kusto_connection_string)
+      @kusto_client = Java::com.microsoft.azure.kusto.ingest.IngestClientFactory.createClient(kusto_connection_string)
 
-      @ingestion_properties = Java::KustoIngestionProperties.new(database, table)
+      @ingestion_properties = Java::com.microsoft.azure.kusto.ingest.IngestionProperties.new(database, table)
       @ingestion_properties.setJsonMappingName(mapping)
 
       @delete_local = delete_local
@@ -48,12 +48,12 @@ def validate_config(database, table, mapping)
       end
 
       if table =~ FIELD_REF
-        @logger.error('table config value should not be dynamic.', database)
+        @logger.error('table config value should not be dynamic.', table)
         raise LogStash::ConfigurationError.new('table config value should not be dynamic.')
       end
 
       if mapping =~ FIELD_REF
-        @logger.error('mapping config value should not be dynamic.', database)
+        @logger.error('mapping config value should not be dynamic.', mapping)
         raise LogStash::ConfigurationError.new('mapping config value should not be dynamic.')
       end
     end
@@ -90,7 +90,8 @@ def upload(path, delete_on_success)
       #   local_ingestion_properties.addJsonMappingName(mapping)
       # end
 
-      @kusto_client.ingestFromSingleFile(path, @ingestion_properties)
+      file_source_info = Java::com.microsoft.azure.kusto.ingest.source.FileSourceInfo.new(path, 0); # 0 - let the sdk figure out the size of the file
+      @kusto_client.ingestFromFile(file_source_info, @ingestion_properties)
 
       File.delete(path) if delete_on_success
 
diff --git a/logstash-output-kusto.gemspec b/logstash-output-kusto.gemspec
@@ -1,12 +1,12 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-kusto'
-  s.version       = '0.1.6'
+  s.version       = '0.2.0'
   s.licenses      = ['Apache-2.0']
-  s.summary       = 'Writes events to Azure KustoDB'
-  s.description   = 'This is a logstash output plugin used to write events to an Azure KustoDB instance'
-  s.homepage      = 'https://github.com/Azure/azure-diagnostics-tools/tree/master/Logstash'
+  s.summary       = 'Writes events to Azure Data Explorer (Kusto)'
+  s.description   = 'This is a logstash output plugin used to write events to an Azure Data Explorer (a.k.a Kusto)'
+  s.homepage      = 'https://github.com/Azure/logstash-output-kusto'
   s.authors       = ['Tamir Kamara']
-  s.email         = 'tamir.kamara@microsoft.com'
+  s.email         = 'nugetkusto@microsoft.com'
   s.require_paths = ['lib']
 
   # Files
diff --git a/spec/outputs/kusto/ingestor_spec.rb b/spec/outputs/kusto/ingestor_spec.rb
@@ -0,0 +1,109 @@
+# encoding: utf-8
+require_relative "../../spec_helpers.rb"
+require 'logstash/outputs/kusto'
+require 'logstash/outputs/kusto/ingestor'
+
+describe LogStash::Outputs::Kusto::Ingestor do
+
+  let(:ingest_url) { "mycluster" }
+  let(:app_id) { "myid" }
+  let(:app_key) { LogStash::Util::Password.new("mykey") }
+  let(:app_tenant) { "mytenant" }
+  let(:database) { "mydatabase" }
+  let(:table) { "mytable" }
+  let(:mapping) { "mymapping" }
+  let(:delete_local) { false }
+  let(:logger) { spy('logger') }
+
+  describe '#initialize' do
+
+    it 'does not throw an error when initializing' do
+      # note that this will cause an internal error since connection is being tried.
+      # however we still want to test that all the java stuff is working as expected
+      expect { 
+        ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, mapping, delete_local, logger)
+        ingestor.stop
+      }.not_to raise_error
+    end
+    
+    dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}']
+
+    context 'doesnt allow database to have some dynamic part' do
+      dynamic_name_array.each do |test_database|
+        it "with database: #{test_database}" do
+          expect {
+            ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, mapping, delete_local, logger)            
+            ingestor.stop
+          }.to raise_error(LogStash::ConfigurationError)          
+        end
+      end
+    end
+
+    context 'doesnt allow table to have some dynamic part' do
+      dynamic_name_array.each do |test_table|
+        it "with database: #{test_table}" do
+          expect {
+            ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, mapping, delete_local, logger)
+            ingestor.stop
+          }.to raise_error(LogStash::ConfigurationError)          
+        end
+      end
+    end
+
+    context 'doesnt allow mapping to have some dynamic part' do
+      dynamic_name_array.each do |test_mapping|
+        it "with database: #{test_mapping}" do
+          expect {
+            ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, test_mapping, delete_local, logger)
+            ingestor.stop
+          }.to raise_error(LogStash::ConfigurationError)          
+        end
+      end
+    end
+
+  end
+
+  # describe 'receiving events' do
+
+  #   context 'with non-zero flush interval' do
+  #     let(:temporary_output_file) { Stud::Temporary.pathname }
+
+  #     let(:event_count) { 100 }
+  #     let(:flush_interval) { 5 }
+
+  #     let(:events) do
+  #       event_count.times.map do |idx|
+  #         LogStash::Event.new('subject' => idx)
+  #       end
+  #     end
+
+  #     let(:output) { described_class.new(options.merge( {'path' => temporary_output_file, 'flush_interval' => flush_interval, 'delete_temp_files' => false } )) }
+
+  #     before(:each) { output.register }
+      
+  #     after(:each) do
+  #       output.close
+  #       File.exist?(temporary_output_file) && File.unlink(temporary_output_file)
+  #       File.exist?(temporary_output_file + '.kusto') && File.unlink(temporary_output_file + '.kusto')
+  #     end
+
+  #     it 'eventually flushes without receiving additional events' do
+  #       output.multi_receive_encoded(events)
+
+  #       # events should not all be flushed just yet...
+  #       expect(File.read(temporary_output_file)).to satisfy("have less than #{event_count} lines") do |contents|
+  #         contents && contents.lines.count < event_count
+  #       end
+
+  #       # wait for the flusher to run...
+  #       sleep(flush_interval + 1)
+
+  #       # events should all be flushed
+  #       expect(File.read(temporary_output_file)).to satisfy("have exactly #{event_count} lines") do |contents|
+  #         contents && contents.lines.count == event_count
+  #       end
+  #     end
+  #   end
+
+  # end
+end
diff --git a/spec/outputs/kusto_spec.rb b/spec/outputs/kusto_spec.rb
@@ -0,0 +1,54 @@
+# encoding: utf-8
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/kusto'
+require 'logstash/codecs/plain'
+require 'logstash/event'
+
+describe LogStash::Outputs::Kusto do
+
+  let(:options) { { "path" => "./kusto_tst/%{+YYYY-MM-dd-HH-mm}",
+    "ingest_url" => "mycluster",
+    "app_id" => "myid",
+    "app_key" => "mykey",
+    "app_tenant" => "mytenant", 
+    "database" => "mydatabase",
+    "table" => "mytable",
+    "mapping" => "mymapping"
+  } }
+
+  describe '#register' do
+
+    it 'doesnt allow the path to start with a dynamic string' do
+      kusto = described_class.new(options.merge( {'path' => '/%{name}'} ))
+      expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
+      kusto.close
+    end
+
+    it 'path must include a dynamic string to allow file rotation' do
+      kusto = described_class.new(options.merge( {'path' => '/{name}'} ))
+      expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
+      kusto.close
+    end
+
+
+    dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}']
+
+    context 'doesnt allow the root directory to have some dynamic part' do
+      dynamic_name_array.each do |test_path|
+         it "with path: #{test_path}" do
+           kusto = described_class.new(options.merge( {'path' => test_path} ))
+           expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
+           kusto.close
+         end
+       end
+    end
+
+    it 'allow to have dynamic part after the file root' do
+      kusto = described_class.new(options.merge({'path' => '/tmp/%{name}'}))
+      expect { kusto.register }.not_to raise_error
+      kusto.close
+    end
+
+  end
+  
+end
diff --git a/spec/spec_helpers.rb b/spec/spec_helpers.rb
@@ -0,0 +1,5 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/logging/logger"
+
+LogStash::Logging::Logger::configure_logging("debug")

Original file line number	Diff line number	Diff line change
`@@ -11,6 +11,15 @@`
`11`	`11`	`"kind": "build",`
`12`	`12`	`"isDefault": true`
`13`	`13`	`}`
	`14`	`+ },`
	`15`	`+ {`
	`16`	`+ "label": "Run tests",`
	`17`	`+ "type": "shell",`
	`18`	`+ "command": "bundle exec rspec spec",`
	`19`	`+ "group": {`
	`20`	`+ "kind": "build",`
	`21`	`+ "isDefault": true`
	`22`	`+ }`
`14`	`23`	`}`
`15`	`24`	`]`
`16`	`25`	`}`