Skip to content

Commit 657b077

Browse files
ppalmesppalmes
committed
refactor
1 parent c01d29a commit 657b077

File tree

10 files changed

+147
-76
lines changed

10 files changed

+147
-76
lines changed

‎AutoAD/Project.toml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
name = "AutoAD"
22
uuid = "a90a1e1f-13cb-41ec-be1f-24b06fd2b474"
3-
authors = ["Paulito Palmes, PhD <[email protected]>"]
43
version = "0.1.0"
4+
authors = ["Paulito Palmes, PhD <[email protected]>"]
55

66
[deps]
77
AMLPipelineBase = "e3c3008a-8869-4d53-9f34-c96f99c8a2b6"
8+
ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
89
AutoMLPipeline = "08437348-eef5-4817-bc1b-d4e9459680d6"
910
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
1011
CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"
@@ -22,6 +23,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
2223

2324
[compat]
2425
AMLPipelineBase = "0.1.18"
26+
ArgParse = "1.2.0"
2527
AutoMLPipeline = "0.4.6"
2628
CSV = "0.10.15"
2729
CondaPkg = "0.2.30"

‎AutoAD/src/AutoAD.jl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ using AMLPipelineBase: AbsTypes, Utils
1515

1616
export get_iris
1717

18+
include("main.jl")
1819

1920
function get_iris()
2021
iris = CSV.read(joinpath(Base.@__DIR__, "../../data", "iris.csv"), DataFrame)

‎AutoAD/src/automlflowtsprediction.jl‎

Lines changed: 3 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
module AutoMLFlowTSPredictions
2+
23
using PDFmerger: append_pdf!
34
using Plots
45
using Statistics
@@ -146,50 +147,9 @@ function mlftsdriver()
146147

147148
#X = vcat(5 * cos.(-10:10), sin.(-30:30), 3 * cos.(-10:10), 2 * tan.(-10:10), sin.(-30:30)) |> x -> DataFrame([x], :auto)
148149

149-
# test all voting percent
150150
mlfas = AutoMLFlowTSPrediction(Dict(:url => url))
151-
fit!(mlfas, X)
152-
#println(Yc |> x -> first(x, 5))
153-
154-
# # test specific votepercent
155-
# mlvad = AutoMLFlowTSPrediction(Dict(:url => url, :impl_args => Dict(:votepercent => 0.3)))
156-
# Yc = fit_transform!(mlvad, X)
157-
# println(Yc |> x -> first(x, 5))
158-
#
159-
# # override default votepercent
160-
# mlfas = AutoMLFlowTSPrediction(Dict(:url => url))
161-
# mlfas.model[:automodel](; votepercent=0.5)
162-
# Yc = fit_transform!(mlfas, X)
163-
# println(Yc |> x -> first(x, 5))
164-
#
165-
#
166-
# mlfas = AutoMLFlowTSPrediction(Dict(:url => url))
167-
# mlfas.model[:automodel](; votepercent=0.0)
168-
# Yc = fit_transform!(mlfas, X)
169-
# println(Yc |> x -> first(x, 5))
170-
#
171-
# ## test prediction using exisiting trained model from artifacts
172-
# #### alternative 1 to use trained model for transform
173-
# mlvad = AutoMLFlowTSPrediction(Dict(:url => url))
174-
# Yc = fit_transform!(mlvad, X)
175-
# run_id = mlvad.model[:run_id]
176-
# newmlad = AutoMLFlowTSPrediction(Dict(:run_id => run_id, :url => url, :impl_args => Dict(:votepercent => 0.5)))
177-
# newmlad.model[:automodel](; votepercent=0.2)
178-
# Yn = transform!(newmlad, X)
179-
# println(Yn |> x -> first(x, 5))
180-
#
181-
# ## alternative 2 to use trained model for transform
182-
# mlvad = AutoMLFlowTSPrediction(Dict(:url => url))
183-
# Yc = fit_transform!(mlvad, X)
184-
# run_id = mlvad.model[:run_id]
185-
# votepercent = 0.3
186-
# newmlad = AutoMLFlowTSPrediction(Dict(:url => url))
187-
# newmlad(; run_id)
188-
# newmlad.model[:automodel](; votepercent)
189-
# Yn = transform!(newmlad, X)
190-
# println(Yn |> x -> first(x, 5))
191-
192-
return nothing
151+
pred=fit_transform!(mlfas, X)
152+
return pred
193153
end
194154

195155
end

‎AutoAD/src/main.jl‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
using AutoAD
2+
using ArgParse
3+
using CSV
4+
using DataFrames
5+
using Statistics
6+
7+
8+
function parse_commandline()
9+
s = ArgParseSettings()
10+
@add_arg_table! s begin
11+
"--url", "-u"
12+
help = "mlflow server url"
13+
arg_type = String
14+
default = "http://localhost:8081"
15+
"--output_file", "-o"
16+
help = "output location"
17+
arg_type = String
18+
default = "NONE"
19+
"--learner", "-l"
20+
help = "learner"
21+
arg_type = String
22+
default = "auto"
23+
"--forecast_horizon", "-f"
24+
help = "forecast horizon"
25+
arg_type = Int64
26+
default = 10
27+
"--runid", "-r"
28+
help = "runid of experiment for trained model"
29+
arg_type = String
30+
default = "NONE"
31+
"--predict_only", "-p"
32+
help = "no training, predict only"
33+
action = :store_true
34+
"csvfile"
35+
help = "input csv file"
36+
required = true
37+
end
38+
return parse_args(s; as_symbols=true)
39+
end
40+
41+
function doprediction_only(args::Dict)
42+
fname = args[:csvfile]
43+
X = CSV.read(fname, DataFrame)
44+
run_id = args[:runid]
45+
url = args[:url]
46+
predtype = args[:prediction_type]
47+
mlf=AutoMLFlowTSPrediction((Dict(:rund_id=>run_id,:url=>url)))
48+
Yn = transform!(mlf, X)
49+
ofile = args[:output_file]
50+
if ofile != "NONE"
51+
open(ofile, "w") do stfile
52+
println(stfile, "prediction: $Yn")
53+
end
54+
end
55+
println(stdout, "prediction: $Yn")
56+
return Yn
57+
end
58+
59+
function dotrainandpredict(args::Dict)
60+
url = args[:url]
61+
learner=args[:learner]
62+
forecast_horizon = args[:forecast_horizon]
63+
fname = args[:csvfile]
64+
df = CSV.read(fname, DataFrame)
65+
X = df[:, 1:1]
66+
autots = AutoMLFlowTSPrediction(Dict(:url => url, :impl_args=>Dict(:forecast_horizon=>forecast_horizon,:learner=>learner)))
67+
Yc = fit_transform!(autots, X)
68+
println("output:", Yc |> x -> first(x, 5))
69+
return Yc
70+
end
71+
72+
function @main(MyARGS)
73+
ARGS = parse_commandline()
74+
if ARGS[:predict_only] == true
75+
# predict only using run_id of model in the artifact
76+
doprediction_only(ARGS)
77+
else
78+
# train and predict
79+
dotrainandpredict(ARGS)
80+
end
81+
end

‎DockerizedAutoAD/Project.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,6 @@ Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
1717
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
1818
StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
1919
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
20+
21+
[sources]
22+
AutoAD = {path = "../AutoAD"}

‎DockerizedAutoML/Project.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,6 @@ Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
1717
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
1818
StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
1919
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
20+
21+
[sources]
22+
AutoAI = {path = "../AutoAI"}

‎DockerizedAutoML/main.jl‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,7 @@ function doprediction_only(args::Dict)
9797
X = CSV.read(fname, DataFrame)
9898
run_id = args[:runid]
9999
url = args[:url]
100-
mlf =
101-
predtype = args[:prediction_type]
100+
predtype = args[:prediction_type]
102101
mlf = if predtype == "classification"
103102
AutoMLFlowClassification(Dict(:run_id => run_id, :url => url))
104103
elseif predtype == "regression"

‎DockerizedAutoTS/Project.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,6 @@ Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
1717
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
1818
StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
1919
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
20+
21+
[sources]
22+
AutoAD = {path = "../AutoAD"}

‎DockerizedAutoTS/main.jl‎

Lines changed: 44 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ using ArgParse
33
using CSV
44
using DataFrames
55
using Statistics
6-
using AutoAD
76

87

98
function parse_commandline()
@@ -12,23 +11,26 @@ function parse_commandline()
1211
"--url", "-u"
1312
help = "mlflow server url"
1413
arg_type = String
15-
default = "http://localhost:8080"
16-
"--prediction_type", "-t"
17-
help = "timeseriesprediction, anomalydetection"
18-
arg_type = String
19-
default = "anomalydetection"
14+
default = "http://localhost:8081"
2015
"--output_file", "-o"
2116
help = "output location"
2217
arg_type = String
2318
default = "NONE"
24-
"--votepercent", "-v"
25-
help = "votepercent for anomalydetection ensembles"
26-
arg_type = Float64
27-
default = 0.0
28-
"--runid"
19+
"--learner", "-l"
20+
help = "learner"
21+
arg_type = String
22+
default = "auto"
23+
"--forecast_horizon", "-f"
24+
help = "forecast horizon"
25+
arg_type = Int64
26+
default = 10
27+
"--runid", "-r"
2928
help = "runid of experiment for trained model"
3029
arg_type = String
3130
default = "NONE"
31+
"--predict_only", "-p"
32+
help = "no training, predict only"
33+
action = :store_true
3234
"csvfile"
3335
help = "input csv file"
3436
required = true
@@ -37,30 +39,47 @@ function parse_commandline()
3739
end
3840

3941
const _cliargs = parse_commandline()
42+
_cliargs[:csvfile]="./../AutoAD/data/node_cpu_ratio_rate_5m_1d_1m.csv"
43+
4044

41-
function autoadmode(args::Dict)
45+
function doprediction_only(args::Dict)
46+
fname = args[:csvfile]
47+
X = CSV.read(fname, DataFrame)
48+
run_id = args[:runid]
4249
url = args[:url]
43-
votepercent = args[:votepercent]
50+
predtype = args[:prediction_type]
51+
mlf=AutoMLFlowTSPrediction((Dict(:rund_id=>run_id,:url=>url)))
52+
Yn = transform!(mlf, X)
53+
ofile = args[:output_file]
54+
if ofile != "NONE"
55+
open(ofile, "w") do stfile
56+
println(stfile, "prediction: $Yn")
57+
end
58+
end
59+
println(stdout, "prediction: $Yn")
60+
return Yn
61+
end
62+
63+
function dotrainandpredict(args::Dict)
64+
url = args[:url]
65+
learner=args[:learner]
66+
forecast_horizon = args[:forecast_horizon]
4467
fname = args[:csvfile]
4568
df = CSV.read(fname, DataFrame)
4669
X = df[:, 1:1]
47-
autoad = AutoMLFlowAnomalyDetection(Dict(:url => url, :impl_args=>Dict(:votepercent=>votepercent)))
48-
Yc = fit_transform!(autoad, X)
70+
autots = AutoMLFlowTSPrediction(Dict(:url => url, :impl_args=>Dict(:forecast_horizon=>forecast_horizon,:learner=>learner)))
71+
Yc = fit_transform!(autots, X)
4972
println("output:", Yc |> x -> first(x, 5))
5073
return Yc
5174
end
5275

53-
function autotsmode(args::Dict)
54-
end
55-
5676
function main(args)
57-
predtype = args[:prediction_type]
58-
if predtype == "anomalydetection"
59-
autoadmode(args)
60-
elseif predtype == "timeseriesprediction"
61-
autotsmode(args)
62-
else
63-
@error "check cli arguments: $args"
77+
if args[:predict_only] == true
78+
# predict only using run_id of model in the artifact
79+
doprediction_only(args)
80+
else
81+
# train and predict
82+
dotrainandpredict(args)
6483
end
6584
end
6685

‎DockerizedAutoTS/run.sh‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
docker build -t automlad --platform=linux/amd64 .
2-
docker run -it --rm --platform=linux/amd64 automlad
1+
docker build -t automlts:v1.0 --platform=linux/amd64 .
2+
docker run -it --rm --platform=linux/amd64 automlai
33

4-
# julia --project -- ./main.jl -c high -t regression -f 3 -w 7 iris_reg.csv
5-
# julia --project -- ./main.jl -c low -t classification -f 3 -w 3 iris.csv
6-
# julia --project -- ./main.jl -c low -t anomalydetection iris.csv
74
# podman run -it --rm --platform=linux/amd64 localhost/automlai -u http://spendor2.sl.cloud9.ibm.com:30412 iris.csv
85
# podman run -it --rm --platform=linux/amd64 localhost/automlad -u http://spendor3.sl.cloud9.ibm.com:30412 ../AutoAD/data/node_cpu_ratio_rate_5m_1d_1m.csv
96
# argo -n argo submit --from clusterworkflowtemplate/automlad-template -p votepercent=0.0 -p input=node_cpu_ratio_rate_5m_1d_1m.csv -p predictiontype=anomalydetection --watch --log
107
# docker run -it --rm --platform=linux/amd64 -v ${HOME}/phome/julia/AutoMLPipeline.jl/AutoAD/data/:/data/ ppalmes/automlad:v2.0 -v 0.0 -u http://mlflow.isiath.duckdns.org:8082 /data/node_cpu_ratio_rate_5m_1d_1m.csv
8+
9+
julia --project ./main.jl -p -f 20 -r 9929adf41952406188c500b19e1e73ab ./../AutoAD/data/node_cpu_ratio_rate_5m_1d_1m.csv
10+
julia -m AutoAD -f 20 -u http://localhost:8081 ./../AutoAD/data/node_cpu_ratio_rate_5m_1d_1m.csv

0 commit comments

Comments
 (0)