From a1cf5dbd7657a4fc122c83893ee06f39162e751d Mon Sep 17 00:00:00 2001 From: Ralf Mueller Date: Thu, 30 Mar 2023 21:25:44 +0200 Subject: [PATCH 1/4] year++ in copyright notice --- ruby/lib/cdo.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/lib/cdo.rb b/ruby/lib/cdo.rb index 157c1c5..80e699b 100644 --- a/ruby/lib/cdo.rb +++ b/ruby/lib/cdo.rb @@ -10,7 +10,7 @@ class Hash alias :include? :has_key? end -# Copyright 2011-2022 Ralf Mueller, ralf.mueller@dkrz.de +# Copyright 2011-2023 Ralf Mueller, ralf.mueller@dkrz.de # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: From 1fd8126bb4182af5fd1d09ab86d85a0b580834e3 Mon Sep 17 00:00:00 2001 From: Ralf Mueller Date: Thu, 30 Mar 2023 21:26:22 +0200 Subject: [PATCH 2/4] Separate static info from the binary from the main class --- ruby/lib/info.rb | 118 +++++++++++++++++++++++++++++++++++++ ruby/test/test_cdo_info.rb | 33 +++++++++++ 2 files changed, 151 insertions(+) create mode 100644 ruby/lib/info.rb create mode 100644 ruby/test/test_cdo_info.rb diff --git a/ruby/lib/info.rb b/ruby/lib/info.rb new file mode 100644 index 0000000..73e5e4a --- /dev/null +++ b/ruby/lib/info.rb @@ -0,0 +1,118 @@ +require 'semverse' +require 'json' +module CdoInfo + # hardcoded fallback list of output operators - from 1.8.0 there is an + # options for this: --operators_no_output + # this list works for cdo-1.6.4 + NoOutputOperators = %w[cdiread cmor codetab conv_cmor_table diff diffc diffn + diffp diffv dump_cmor_table dumpmap filedes gmtcells gmtxyz gradsdes griddes + griddes2 gridverify info infoc infon infop infos infov map ncode ndate + ngridpoints ngrids nlevel nmon npar ntime nvar nyear output outputarr + outputbounds outputboundscpt outputcenter outputcenter2 outputcentercpt + outputext outputf outputfld outputint outputkey outputsrv outputtab outputtri + outputts outputvector outputvrml outputxyz pardes partab partab2 seinfo + seinfoc seinfon seinfop showattribute showatts showattsglob showattsvar + showcode showdate showformat showgrid showlevel showltype showmon showname + showparam showstdname showtime showtimestamp showunit showvar showyear sinfo + sinfoc sinfon sinfop sinfov spartab specinfo tinfo vardes vct vct2 verifygrid + vlist xinfon zaxisdes] + TwoOutputOperators = %w[trend samplegridicon mrotuv eoftime + eofspatial eof3dtime eof3dspatial eof3d eof complextorect complextopol] + MoreOutputOperators = %w[distgrid eofcoeff eofcoeff3d intyear scatter splitcode + splitday splitgrid splithour splitlevel splitmon splitname splitparam splitrec + splitseas splitsel splittabnum splitvar splityear splityearmon splitzaxis] + def CdoInfo.version(executable) + info = IO.popen(executable+' -V 2>&1').readlines.first + info.split(' ').grep(%r{\d+\.\d+.*})[0].to_s + end + " get the CDO version " + def CdoInfo.semversion(executable) + Semverse::Version.new(CdoInfo.version(executable)) + end + + " get supported filetypes of the binary and other configuration data" + def CdoInfo.config(executable) + config = {} + config.default(false) + + if Semverse::Version.new('1.9.3') < CdoInfo.semversion(executable) then + config.merge!(JSON.parse(IO.popen(executable + " --config all").read.chomp)) + config.each {|k,v| config[k] = ('yes' == v) ? true : false} + else + warn "Cannot check configuration of the binary!" + warn "Please check manually with '#{executable} -V'" + end + config + end + + " get an infentory for the operators provided by the executable " + def CdoInfo.operators(executable) #{{{ + operators = {} + + version = CdoInfo.semversion(executable) + + # little side note: the option --operators_no_output works in 1.8.0 and + # 1.8.2, but not in 1.9.0, from 1.9.1 it works again + case + when version < Semverse::Version.new('1.7.2') then + cmd = executable + ' 2>&1' + help = IO.popen(cmd).readlines.map {|l| l.chomp.lstrip} + if 5 >= help.size + warn "Operators could not get listed by running the CDO binary (#{executable})" + pp help if @debug + exit + end + + _operators = help[(help.index("Operators:")+1)..help.index(help.find {|v| + v =~ /CDO version/ + }) - 2].join(' ').split + + # build up operator inventory + # default is 1 output stream + _operators.each {|op| operators[op] = 1 } + operators.each {|op,_| + operators[op] = 0 if NoOutputOperators.include?(op) + operators[op] = 2 if TwoOutputOperators.include?(op) + operators[op] = -1 if MoreOutputOperators.include?(op) + } + + when (version < Semverse::Version.new('1.8.0') or Semverse::Version.new('1.9.0') == version) then + cmd = "#{executable} --operators" + _operators = IO.popen(cmd).readlines.map {|l| l.split(' ').first } + + _operators.each {|op| operators[op] = 1 } + operators.each {|op,_| + operators[op] = 0 if NoOutputOperators.include?(op) + operators[op] = 2 if TwoOutputOperators.include?(op) + operators[op] = -1 if MoreOutputOperators.include?(op) + } + + + when version < Semverse::Version.new('1.9.3') then + cmd = "#{executable} --operators" + _operators = IO.popen(cmd).readlines.map {|l| l.split(' ').first } + cmd = "#{executable} --operators_no_output" + _operatorsNoOutput = IO.popen(cmd).readlines.map {|l| l.split(' ').first } + + # build up operator inventory + _operators.each {|op| operators[op] = 1 } + _operatorsNoOutput.each {|op| operators[op] = 0} + operators.each {|op,_| + operators[op] = 0 if _operatorsNoOutput.include?(op) + operators[op] = 2 if TwoOutputOperators.include?(op) + operators[op] = -1 if MoreOutputOperators.include?(op) + } + + else + cmd = "#{executable} --operators" + operators = {} + IO.popen(cmd).readlines.map {|line| + lineContent = line.chomp.split(' ') + name = lineContent[0] + iCounter, oCounter = lineContent[-1][1..-1].split('|') + operators[name] = oCounter.to_i + } + end + return operators + end +end diff --git a/ruby/test/test_cdo_info.rb b/ruby/test/test_cdo_info.rb new file mode 100644 index 0000000..574dbcf --- /dev/null +++ b/ruby/test/test_cdo_info.rb @@ -0,0 +1,33 @@ +$:.unshift File.join(File.dirname(__FILE__),"..","lib") +require 'info' +require 'minitest/autorun' +#=============================================================================== +def rm(files); files.each {|f| FileUtils.rm(f) if File.exist?(f)};end + +class TestCdoInfo < Minitest::Test + def test_version + version = CdoInfo.version('/usr/bin/cdo') + assert_equal('2.1.1',version) + version = CdoInfo.semversion('/usr/bin/cdo') + assert_equal(Semverse::Version.new('2.1.1'),version) + end + def test_config + config = CdoInfo.config('cdo') + expectedConfig = {"has-cgribex"=>true, "has-cmor"=>false, "has-ext"=>true, + "has-grb"=>true, "has-grb1"=>true, "has-grb2"=>true, + "has-hdf5"=>true, "has-ieg"=>true, "has-magics"=>true, + "has-nc"=>true, "has-nc2"=>true, "has-nc4"=>true, + "has-nc4c"=>true, "has-nc5"=>true, "has-nczarr"=>true, + "has-openmp"=>true, "has-proj"=>true, "has-srv"=>true, + "has-threads"=>true, "has-wordexp"=>true} + assert_equal(config,expectedConfig) + end + def test_operators + operators = CdoInfo.operators('cdo') + pp operators + assert_equal(operators["map"],-1) + assert_equal(operators["select"],1) + assert_equal(operators["fldmean"],1) + assert_equal(operators["infov"],1) + end +end From 2719df0dc3cb5739086a91d8b8629f8bcc18dd43 Mon Sep 17 00:00:00 2001 From: Ralf Mueller Date: Fri, 31 Mar 2023 09:29:01 +0200 Subject: [PATCH 3/4] add files for new API --- ruby/lib/cdoNG.rb | 20 ++++++++++++++++++++ ruby/test/test_cdo_ng.rb | 29 +++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 ruby/lib/cdoNG.rb create mode 100644 ruby/test/test_cdo_ng.rb diff --git a/ruby/lib/cdoNG.rb b/ruby/lib/cdoNG.rb new file mode 100644 index 0000000..56739a0 --- /dev/null +++ b/ruby/lib/cdoNG.rb @@ -0,0 +1,20 @@ +class CdoNG + def initialize(cdo: 'cdo', + returnFalseOnError: false, + returnNilOnError: false, + forceOutput: true, + env: {}, + debug: false, + tempdir: Dir.tmpdir, + logging: false, + logFile: StringIO.new) + end + def run(output: nil, + returnArray: false, + returnMaArray: false, + force: true, + env: {}, + debug: false) + return true + end +end diff --git a/ruby/test/test_cdo_ng.rb b/ruby/test/test_cdo_ng.rb new file mode 100644 index 0000000..d33fdea --- /dev/null +++ b/ruby/test/test_cdo_ng.rb @@ -0,0 +1,29 @@ +$:.unshift File.join(File.dirname(__FILE__),"..","lib") +require 'cdoNG' + +require 'minitest/autorun' + +Cdo = CdoNG + +#=============================================================================== +def rm(files); files.each {|f| FileUtils.rm(f) if File.exist?(f)};end + + +class TestCdo < Minitest::Test + + DEFAULT_CDO_PATH = 'cdo' + + @@show = ENV.has_key?('SHOW') + @@maintainermode = ENV.has_key?('MAINTAINERMODE') + @@debug = ENV.has_key?('DEBUG') + + parallelize_me! unless @@debug + + def setup + @cdo = Cdo.new + end + + def test_cdo + assert_equal(true,@cdo.run) + end +end From df000000f82309fbe176b8e1ef34284d91d061ad Mon Sep 17 00:00:00 2001 From: Ralf Mueller Date: Fri, 31 Mar 2023 20:15:24 +0200 Subject: [PATCH 4/4] split command execution and getting meta info from the binary + testing --- ruby/lib/cdoNG.rb | 36 ++++++++++++++++++++- ruby/lib/info.rb | 64 ++++++++++++++++++++------------------ ruby/test/test_cdo.rb | 1 + ruby/test/test_cdo_info.rb | 44 ++++++++++++++++++++------ ruby/test/test_cdo_ng.rb | 6 ++++ 5 files changed, 110 insertions(+), 41 deletions(-) diff --git a/ruby/lib/cdoNG.rb b/ruby/lib/cdoNG.rb index 56739a0..828b3ce 100644 --- a/ruby/lib/cdoNG.rb +++ b/ruby/lib/cdoNG.rb @@ -1,5 +1,9 @@ +load 'info.rb' class CdoNG - def initialize(cdo: 'cdo', + attr_reader :options, :cdo, :forceOutput + attr_accessor :returnFalseOnError, :returnNilOnError, :debug, :logging, :logFile, :tempdir + + def initialize(executable: 'cdo', returnFalseOnError: false, returnNilOnError: false, forceOutput: true, @@ -8,6 +12,20 @@ def initialize(cdo: 'cdo', tempdir: Dir.tmpdir, logging: false, logFile: StringIO.new) + + @executable = executable + @returnFalseOnError = returnFalseOnError + @returnNilOnError = returnNilOnError + @forceOutput = forceOutput + @env = env + @debug = debug + @tempdir = tempdir + @logging = logging + @logFile = logFile + + @commands = [] + @operators = CdoInfo.operators(@executable) + end def run(output: nil, returnArray: false, @@ -17,4 +35,20 @@ def run(output: nil, debug: false) return true end + def method_missing(sym, *args, **kwargs) + operatorName = sym.to_s + puts "Operator #{operatorName} is called" if @debug + + # exit eary on missing operator + unless @operators.include?(operatorName) + return false if @returnFalseOnError + raise ArgumentError,"Operator #{operatorName} not found" + end + + # check of kwargs, raise error on missing or unknown + # output is invalid + # options might be valid (special operatiors need options attached) + # + # append operator incl parameters + end end diff --git a/ruby/lib/info.rb b/ruby/lib/info.rb index 73e5e4a..d45e906 100644 --- a/ruby/lib/info.rb +++ b/ruby/lib/info.rb @@ -1,5 +1,8 @@ require 'semverse' require 'json' +require 'open3' + +# Support module that should encapsulate basic operations with the binary module CdoInfo # hardcoded fallback list of output operators - from 1.8.0 there is an # options for this: --operators_no_output @@ -21,11 +24,14 @@ module CdoInfo MoreOutputOperators = %w[distgrid eofcoeff eofcoeff3d intyear scatter splitcode splitday splitgrid splithour splitlevel splitmon splitname splitparam splitrec splitseas splitsel splittabnum splitvar splityear splityearmon splitzaxis] + + "return CDI versio as string" def CdoInfo.version(executable) info = IO.popen(executable+' -V 2>&1').readlines.first info.split(' ').grep(%r{\d+\.\d+.*})[0].to_s end - " get the CDO version " + + "return semantiv version of CDO" def CdoInfo.semversion(executable) Semverse::Version.new(CdoInfo.version(executable)) end @@ -45,46 +51,37 @@ def CdoInfo.config(executable) config end + "Check if the --operators is present" + def CdoInfo.hasOperatorsOption(executable) + log, status = Open3.capture2e("#{executable} --operators") + return (0 == status) + end + " get an infentory for the operators provided by the executable " + " this depends on the availability of the --operators option " def CdoInfo.operators(executable) #{{{ operators = {} + unless CdoInfo.hasOperatorsOption(executable) then + warn "Cannot create database of operators!" + exit(1) + end + version = CdoInfo.semversion(executable) # little side note: the option --operators_no_output works in 1.8.0 and # 1.8.2, but not in 1.9.0, from 1.9.1 it works again case - when version < Semverse::Version.new('1.7.2') then - cmd = executable + ' 2>&1' - help = IO.popen(cmd).readlines.map {|l| l.chomp.lstrip} - if 5 >= help.size - warn "Operators could not get listed by running the CDO binary (#{executable})" - pp help if @debug - exit - end - - _operators = help[(help.index("Operators:")+1)..help.index(help.find {|v| - v =~ /CDO version/ - }) - 2].join(' ').split - - # build up operator inventory - # default is 1 output stream - _operators.each {|op| operators[op] = 1 } - operators.each {|op,_| - operators[op] = 0 if NoOutputOperators.include?(op) - operators[op] = 2 if TwoOutputOperators.include?(op) - operators[op] = -1 if MoreOutputOperators.include?(op) - } - when (version < Semverse::Version.new('1.8.0') or Semverse::Version.new('1.9.0') == version) then cmd = "#{executable} --operators" _operators = IO.popen(cmd).readlines.map {|l| l.split(' ').first } _operators.each {|op| operators[op] = 1 } operators.each {|op,_| - operators[op] = 0 if NoOutputOperators.include?(op) - operators[op] = 2 if TwoOutputOperators.include?(op) - operators[op] = -1 if MoreOutputOperators.include?(op) + oCounter = 0 if NoOutputOperators.include?(op) + oCounter = 2 if TwoOutputOperators.include?(op) + oCounter = -1 if MoreOutputOperators.include?(op) + operators[op] = {:in => 1, :out => oCounter} } @@ -98,9 +95,10 @@ def CdoInfo.operators(executable) #{{{ _operators.each {|op| operators[op] = 1 } _operatorsNoOutput.each {|op| operators[op] = 0} operators.each {|op,_| - operators[op] = 0 if _operatorsNoOutput.include?(op) - operators[op] = 2 if TwoOutputOperators.include?(op) - operators[op] = -1 if MoreOutputOperators.include?(op) + oCounter = 0 if _operatorsNoOutput.include?(op) + oCounter = 2 if TwoOutputOperators.include?(op) + oCounter = -1 if MoreOutputOperators.include?(op) + operators[op] = {:in => 1, :out => oCounter} } else @@ -110,9 +108,15 @@ def CdoInfo.operators(executable) #{{{ lineContent = line.chomp.split(' ') name = lineContent[0] iCounter, oCounter = lineContent[-1][1..-1].split('|') - operators[name] = oCounter.to_i + operators[name] = {:in => iCounter.to_i , :out => oCounter.to_i} } end return operators end + # check if the CDO binary is present and works + def CdoInfo.works?(executable) + status = system("#{executable} -V >/dev/null 2>&1") + fullpath = File.exist?(executable) and File.executable?(executable) + return (status or fullpath) + end end diff --git a/ruby/test/test_cdo.rb b/ruby/test/test_cdo.rb index 348ef73..3e87abf 100644 --- a/ruby/test/test_cdo.rb +++ b/ruby/test/test_cdo.rb @@ -53,6 +53,7 @@ def test_hasCdo assert(@cdo.hasCdo) if File.exist?(@cdo.cdo) end def test_getOperators + pp @cdo.operators %w[seq random stdatm info showlevel sinfo remap geopotheight mask topo thicknessOfLevels].each {|op| if ["thicknessOfLevels"].include?(op) assert(@cdo.respond_to?(op),"Operator '#{op}' not found") diff --git a/ruby/test/test_cdo_info.rb b/ruby/test/test_cdo_info.rb index 574dbcf..15afd80 100644 --- a/ruby/test/test_cdo_info.rb +++ b/ruby/test/test_cdo_info.rb @@ -1,33 +1,57 @@ $:.unshift File.join(File.dirname(__FILE__),"..","lib") require 'info' require 'minitest/autorun' +require 'semverse' #=============================================================================== def rm(files); files.each {|f| FileUtils.rm(f) if File.exist?(f)};end +#=============================================================================== +EXECUTABLE = ENV.has_key?('CDO') ? ENV['CDO'] : 'cdo' +#=============================================================================== class TestCdoInfo < Minitest::Test def test_version - version = CdoInfo.version('/usr/bin/cdo') + version = CdoInfo.version(EXECUTABLE) assert_equal('2.1.1',version) - version = CdoInfo.semversion('/usr/bin/cdo') + version = CdoInfo.semversion(EXECUTABLE) assert_equal(Semverse::Version.new('2.1.1'),version) end def test_config - config = CdoInfo.config('cdo') + config = CdoInfo.config(EXECUTABLE) +# if Semverse::Version.new('2.1.1') == CdoInfo.semversion(@executable) then expectedConfig = {"has-cgribex"=>true, "has-cmor"=>false, "has-ext"=>true, "has-grb"=>true, "has-grb1"=>true, "has-grb2"=>true, "has-hdf5"=>true, "has-ieg"=>true, "has-magics"=>true, "has-nc"=>true, "has-nc2"=>true, "has-nc4"=>true, "has-nc4c"=>true, "has-nc5"=>true, "has-nczarr"=>true, "has-openmp"=>true, "has-proj"=>true, "has-srv"=>true, - "has-threads"=>true, "has-wordexp"=>true} + "has-threads"=>true, "has-wordexp"=>true, "has-hirlam_extensions"=>false} +# else if false then +# end assert_equal(config,expectedConfig) end def test_operators - operators = CdoInfo.operators('cdo') - pp operators - assert_equal(operators["map"],-1) - assert_equal(operators["select"],1) - assert_equal(operators["fldmean"],1) - assert_equal(operators["infov"],1) + operators = CdoInfo.operators(EXECUTABLE) + + # check for specific operators properties + # maps abritrary inputs and 0 output + assert_equal(-1,operators["map"][:in]) + assert_equal(0,operators["map"][:out]) + + assert_equal(-1,operators["select"][:in]) + assert_equal(1,operators["select"][:out]) + + assert_equal(1,operators["fldmean"][:in]) + assert_equal(1,operators["fldmean"][:out]) + + + assert_equal(1,operators["zaxisdes"][:in]) + assert_equal(0,operators["zaxisdes"][:out]) + end + + def test_works + assert(CdoInfo.works?('cdo')) + assert(!CdoInfo.works?('cda_____o')) + assert(!CdoInfo.works?('ls')) + assert(CdoInfo.works?(EXECUTABLE)) end end diff --git a/ruby/test/test_cdo_ng.rb b/ruby/test/test_cdo_ng.rb index d33fdea..fbe3e98 100644 --- a/ruby/test/test_cdo_ng.rb +++ b/ruby/test/test_cdo_ng.rb @@ -26,4 +26,10 @@ def setup def test_cdo assert_equal(true,@cdo.run) end + + def test_operator_missing + assert_raises ArgumentError do + @cdo.noexistent(:input => '-for,d') + end + end end