Skip to content

Commit 0cf34f2

Browse files
EleonoraPetzoldEleonoraPetzold
committed
replace missing value close #59
use generic function to get primary data from api
1 parent bef73d1 commit 0cf34f2

File tree

8 files changed

+254
-177
lines changed

8 files changed

+254
-177
lines changed

‎BExIS.Modules.Lui.UI.csproj‎

Lines changed: 82 additions & 60 deletions
Large diffs are not rendered by default.

‎Controllers/LUICalculationController.cs‎

Lines changed: 37 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
using Vaiona.Web.Mvc.Modularity;
2222
using System.Web.Routing;
2323
using Microsoft.AspNet.Identity;
24+
using System.Globalization;
2425

2526
namespace BExIS.Modules.Lui.UI.Controllers
2627
{
@@ -89,7 +90,9 @@ public ActionResult ShowPrimaryData(long datasetID, bool isPublicAccess)
8990
Session["LUICalModel"] = lUIQueryModel;
9091

9192
DataModel model = new DataModel();
92-
DataTable data = DataAccess.GetComponentData(datasetID.ToString(), GetServerInformation());
93+
//get data structureId
94+
long structureId = long.Parse(DataAccess.GetDatasetInfo(datasetID.ToString(), GetServerInformation()).DataStructureId, CultureInfo.InvariantCulture);
95+
DataTable data = DataAccess.GetData(datasetID.ToString(), structureId, GetServerInformation());
9396
if (isPublicAccess)
9497
{
9598
//remove not complete years data for non public access
@@ -152,7 +155,9 @@ public ActionResult CalculateLUI(LUIQueryModel model)
152155
dsId = Models.Settings.get("lui:datasetNewComponentsSet").ToString();
153156
break;
154157
}
155-
DataTable dt_sourceData = DataAccess.GetComponentData(dsId, GetServerInformation());
158+
159+
long structureId = long.Parse(DataAccess.GetDatasetInfo(dsId, GetServerInformation()).DataStructureId, CultureInfo.InvariantCulture);
160+
DataTable dt_sourceData = DataAccess.GetData(dsId, structureId, GetServerInformation());
156161

157162
var results = CalculateLui.DoCalc(model, dt_sourceData);
158163

@@ -189,9 +194,12 @@ public ActionResult PrepareDownloadFile(string mimeType)
189194
DataTable downloadData = new DataTable();
190195

191196
LUIQueryModel model = (LUIQueryModel)Session["LUICalModel"];
192-
197+
193198
if (model.RawVsCalc.SelectedValue == "unstandardized")
194-
downloadData = DataAccess.GetComponentData(model.DownloadDatasetId, GetServerInformation());
199+
{
200+
long structureId = long.Parse(DataAccess.GetDatasetInfo(model.DownloadDatasetId, GetServerInformation()).DataStructureId, CultureInfo.InvariantCulture);
201+
downloadData = DataAccess.GetData(model.DownloadDatasetId, structureId, GetServerInformation());
202+
}
195203
else
196204
downloadData = Session[SESSION_TABLE] as DataTable;
197205

@@ -263,8 +271,6 @@ public ActionResult PrepareDownloadFile(string mimeType)
263271
zip.Save(zipFilePath);
264272
}
265273

266-
267-
268274
// store path in session for further download
269275
if (null == Session[SESSION_FILE])
270276
{
@@ -273,10 +279,6 @@ public ActionResult PrepareDownloadFile(string mimeType)
273279

274280
((Dictionary<string, string>)Session[SESSION_FILE])[mimeType] = zipFilePath;
275281

276-
277-
278-
279-
280282
return Json(new { error = false, mimeType = mimeType }, JsonRequestBehavior.AllowGet);
281283
}
282284

@@ -358,20 +360,31 @@ private bool checkPreconditions()
358360
{
359361
// check for LUI new dataset
360362
bool exists = false;
361-
string luiIdNew = Models.Settings.get("lui:datasetNewComponentsSet").ToString();
362-
var dataNew = DataAccess.GetComponentData(luiIdNew, GetServerInformation());
363-
if (dataNew.Rows.Count == 0)
364-
return exists == false;
363+
try
364+
{
365+
string luiIdNew = Models.Settings.get("lui:datasetNewComponentsSet").ToString();
366+
367+
long structureIdNew = long.Parse(DataAccess.GetDatasetInfo(luiIdNew, GetServerInformation()).DataStructureId, CultureInfo.InvariantCulture);
368+
var dataNew = DataAccess.GetData(luiIdNew, structureIdNew, GetServerInformation());
369+
if (dataNew.Rows.Count == 0)
370+
return exists == false;
365371

366-
string dsdId = Models.Settings.get("lui:datastructureNewComponentsSet").ToString();
372+
string dsdId = Models.Settings.get("lui:datastructureNewComponentsSet").ToString();
367373

368-
// check for LUI old dataset
369-
string luiIdOld = Models.Settings.get("lui:datasetOldComponentsSet").ToString();
370-
var dataOld = DataAccess.GetComponentData(luiIdOld, GetServerInformation());
371-
if (dataOld.Rows.Count == 0)
372-
return exists == false;
374+
// check for LUI old dataset
375+
string luiIdOld = Models.Settings.get("lui:datasetOldComponentsSet").ToString();
376+
long structureIdOld = long.Parse(DataAccess.GetDatasetInfo(luiIdNew, GetServerInformation()).DataStructureId, CultureInfo.InvariantCulture);
373377

374-
int dsdIdOld = (int)Models.Settings.get("lui:datastructureOldComponentsSet");
378+
var dataOld = DataAccess.GetData(luiIdOld, structureIdOld, GetServerInformation());
379+
if (dataOld.Rows.Count == 0)
380+
return exists == false;
381+
382+
int dsdIdOld = (int)Models.Settings.get("lui:datastructureOldComponentsSet");
383+
}
384+
catch(Exception e)
385+
{
386+
throw new Exception("Precondion failed: ",e);
387+
}
375388

376389
// if we came that far, all conditions are met
377390
return true;
@@ -442,7 +455,9 @@ private void RemoveNotComplateYears(DataTable data, List<string> years)
442455
/// <returns>List of years</returns>
443456
private List<CheckboxControlHelper> GetAvailableYears(string datasetId, bool isPublicAccess)
444457
{
445-
DataTable data = DataAccess.GetComponentData(datasetId, GetServerInformation());
458+
//get data structureId
459+
long structureId = long.Parse(DataAccess.GetDatasetInfo(datasetId.ToString(), GetServerInformation()).DataStructureId, CultureInfo.InvariantCulture);
460+
DataTable data = DataAccess.GetData(datasetId, structureId, GetServerInformation());
446461
if (isPublicAccess)
447462
{
448463
var missingData = DataAccess.GetMissingComponentData(GetServerInformation());

‎Controllers/LUIComponentCalculationController.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,9 @@ public ServerInformation GetServerInformation()
216216
private int[] CheckDuplicates(DataTable newCompData, int[] rowIds)
217217
{
218218
string luiIdNew = Models.Settings.get("lui:datasetNewComponentsSet").ToString();
219-
DataTable allCompData = DataAccess.GetComponentData(luiIdNew, GetServerInformation());
219+
long structureId = long.Parse(DataAccess.GetDatasetInfo(luiIdNew, GetServerInformation()).DataStructureId, CultureInfo.InvariantCulture);
220+
221+
DataTable allCompData = DataAccess.GetData(luiIdNew, structureId, GetServerInformation());
220222

221223
List<int> duplicates = new List<int>();
222224
foreach (int id in rowIds)

‎Helper/DataAccess.cs‎

Lines changed: 72 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ namespace BExIS.Modules.Lui.UI.Helper
1919
{
2020
public class DataAccess
2121
{
22-
2322
/// <summary>
2423
/// Get metadata
2524
/// </summary>
@@ -55,10 +54,12 @@ public static DataTable GetData(string datasetId, long structureId, ServerInform
5554
string link = serverInformation.ServerName + "/api/data/" + datasetId;
5655
HttpWebRequest request = WebRequest.Create(link) as HttpWebRequest;
5756
request.Headers.Add("Authorization", "Bearer " + serverInformation.Token);
58-
// request.ContentType = "application/json";
57+
58+
List<ApiDataStatisticModel> statistics = GetMissingValues(datasetId, serverInformation);
5959

6060
DataStructureObject dataStructureObject = GetDataStructure(structureId, serverInformation);
6161

62+
//create datatable using data structure info
6263
DataTable data = new DataTable();
6364
foreach (var variable in dataStructureObject.Variables)
6465
{
@@ -68,7 +69,6 @@ public static DataTable GetData(string datasetId, long structureId, ServerInform
6869
data.Columns.Add(col);
6970
}
7071

71-
string a = "";
7272
try
7373
{
7474
// Get response
@@ -78,124 +78,89 @@ public static DataTable GetData(string datasetId, long structureId, ServerInform
7878
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
7979
using (var csvReader = new CsvReader(reader, CultureInfo.InvariantCulture))
8080
{
81-
8281
var records = csvReader.GetRecords<dynamic>();
8382

8483
foreach (var r in records)
8584
{
86-
8785
var l = Enumerable.ToList(r);
8886

8987
DataRow dr = data.NewRow();
9088
String[] row = new String[4];
89+
9190
for (int j = 0; j < data.Columns.Count; j++)
9291
{
93-
a = l[j].Value;
9492
if (String.IsNullOrEmpty(l[j].Value))
9593
dr[data.Columns[j].ColumnName] = DBNull.Value;
9694
else
97-
dr[data.Columns[j].ColumnName] = l[j].Value;
98-
95+
{
96+
//get missing values from data structure to replace display name with placeholder
97+
DataTable missingValues = statistics.Where(a => a.VariableName == data.Columns[j].ColumnName).Select(a => a.missingValues).FirstOrDefault();
98+
dynamic value = null;
99+
if (missingValues != null)
100+
{
101+
var displayNames = missingValues.AsEnumerable().Select(a => a.Field<string>("displayName")).ToList();
102+
if (displayNames.Contains(l[j].Value))
103+
value = missingValues.AsEnumerable().Where(a => a.Field<string>("displayName") == l[j].Value).Select(a => a.Field<string>("placeholder")).FirstOrDefault();
104+
else
105+
value = l[j].Value;
106+
107+
}
108+
else
109+
value = l[j].Value;
110+
111+
if (data.Columns[j].DataType == typeof(DateTime))
112+
{
113+
var format = dataStructureObject.Variables.Where(e => e.Label == data.Columns[j].ColumnName).FirstOrDefault().DataType;
114+
format = format.Split('-').ToArray()[1];
115+
dr[data.Columns[j].ColumnName] = ParseValue(data.Columns[j].DataType.ToString(), value, format);
116+
}
117+
else
118+
dr[data.Columns[j].ColumnName] = value;
119+
}
99120
}
100121

101122
data.Rows.Add(dr);
102123
}
103124

104-
//JavaScriptSerializer js = new JavaScriptSerializer();
105-
//var objText = reader.ReadToEnd();
106-
//lanuData = (DataTable)JsonConvert.DeserializeObject(objText, (typeof(DataTable)));
107-
108125
response.Close();
109126
}
110127
}
111128
}
112129
catch (Exception e)
113130
{
114-
string t = a;
131+
115132
}
116133

117134

118135
return data;
119136
}
120137

121-
122-
/// <summary>
123-
/// Get comp data
124-
///
125-
/// </summary>
126-
/// <param name="datasetId"></param>
127-
/// <returns>Data table with comp dataset depents on dataset id.</returns>
128-
public static DataTable GetComponentData(string datasetId, ServerInformation serverInformation)
138+
public static List<ApiDataStatisticModel> GetMissingValues(string datasetId, ServerInformation serverInformation)
129139
{
130-
string link = serverInformation.ServerName + "/api/data/" + datasetId;
140+
string link = serverInformation.ServerName + "/api/datastatistic/" + datasetId;
131141
HttpWebRequest request = WebRequest.Create(link) as HttpWebRequest;
132142
request.Headers.Add("Authorization", "Bearer " + serverInformation.Token);
133143

134-
DataTable compData = new DataTable();
135-
DataColumn idCol = new DataColumn("Id");
136-
idCol.DataType = System.Type.GetType("System.Int32");
137-
compData.Columns.Add(idCol);
138-
DataColumn year = new DataColumn("Year");
139-
year.DataType = System.Type.GetType("System.DateTime");
140-
compData.Columns.Add(year);
141-
compData.Columns.Add("Exploratory");
142-
compData.Columns.Add("EP_PlotID");
143-
compData.Columns.Add("isVIP");
144-
compData.Columns.Add("isMIP");
145-
DataColumn totalGrazing = new DataColumn("TotalGrazing");
146-
totalGrazing.DataType = System.Type.GetType("System.Decimal");
147-
compData.Columns.Add(totalGrazing);
148-
DataColumn totalMowing = new DataColumn("TotalMowing");
149-
totalMowing.DataType = System.Type.GetType("System.Decimal");
150-
compData.Columns.Add(totalMowing);
151-
DataColumn totalFertilization = new DataColumn("TotalFertilization");
152-
totalFertilization.DataType = System.Type.GetType("System.Decimal");
153-
compData.Columns.Add(totalFertilization);
144+
List<ApiDataStatisticModel> data = new List<ApiDataStatisticModel>();
154145

155146
try
156147
{
157148
// Get response
158149
using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
159150
{
160-
// Get the response stream
161151
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
162152
{
163-
string line = String.Empty;
164-
string sep = "\t";
165-
String[] row = new String[4];
166-
int count = 0;
167-
int id = 0;
168-
while ((line = reader.ReadLine()) != null)
169-
{
170-
count++;
171-
id++;
172-
if (count > 1)
173-
{
174-
row = line.Split(',');
175-
DataRow dr = compData.NewRow();
176-
//dr["Year"] = DateTime.Parse(row[0]).ToString("yyyy");
177-
dr["id"] = id;
178-
dr["Year"] = row[0];
179-
dr["Exploratory"] = row[1];
180-
dr["EP_PlotID"] = row[2];
181-
dr["isVIP"] = row[3];
182-
dr["isMIP"] = row[4];
183-
dr["TotalGrazing"] = row[5];
184-
dr["TotalMowing"] = row[6];
185-
dr["TotalFertilization"] = row[7];
186-
compData.Rows.Add(dr);
187-
}
188-
}
189-
response.Close();
153+
var objText = reader.ReadToEnd();
154+
data = JsonConvert.DeserializeObject<List<ApiDataStatisticModel>>(objText);
190155
}
191156
}
192157
}
193158
catch (Exception e)
194159
{
195-
160+
string error = "Not data" + e.InnerException;
196161
}
197162

198-
return compData;
163+
return data;
199164
}
200165

201166
/// <summary>
@@ -207,7 +172,8 @@ public static List<MissingComponentData> GetMissingComponentData(ServerInformati
207172
{
208173
List<MissingComponentData> data = new List<MissingComponentData>();
209174
string datasetId = Models.Settings.get("lui:datasetNewComponentsSet").ToString();
210-
DataTable compData = GetComponentData(datasetId, serverInformation);
175+
long structureId = long.Parse(DataAccess.GetDatasetInfo(datasetId, serverInformation).DataStructureId, CultureInfo.InvariantCulture);
176+
DataTable compData = GetData(datasetId, structureId, serverInformation);
211177

212178
//get all years where data rows less then 50, that means not all plots has data
213179
var years = compData.AsEnumerable().GroupBy(x => x.Field<DateTime>("Year")).Where(g => g.Count() < 150).ToList();
@@ -271,7 +237,11 @@ public static DatasetObject GetDatasetInfo(string datasetId, ServerInformation s
271237

272238
return datasetObject;
273239
}
274-
240+
/// <summary>
241+
/// Get data structure
242+
///
243+
/// </summary>
244+
/// <returns> a data structure object</returns>
275245
public static DataStructureObject GetDataStructure(long structId, ServerInformation serverInformation)
276246
{
277247
string link = serverInformation.ServerName + "/api/structures/" + structId;
@@ -301,9 +271,6 @@ public static DataStructureObject GetDataStructure(long structId, ServerInformat
301271
return dataStructureObject;
302272
}
303273

304-
305-
306-
307274
/// <summary>
308275
/// get all ep plot ids from grasland plots
309276
///
@@ -363,6 +330,11 @@ public static List<string> getAllGrasslandPlots(ServerInformation serverInformat
363330
return graslandPlots;
364331
}
365332

333+
/// <summary>
334+
/// upload primary data via api
335+
///
336+
/// </summary>
337+
/// <returns>API response</returns>
366338
public static string Upload(DataApiModel data, ServerInformation serverInformation)
367339
{
368340
string link = serverInformation.ServerName + "/api/Data/";
@@ -386,7 +358,30 @@ public static string Upload(DataApiModel data, ServerInformation serverInformati
386358
}
387359

388360
return result;
361+
}
389362

363+
/// <summary>
364+
/// convert string to given data type
365+
///
366+
/// </summary>
367+
/// <returns>type object</returns>
368+
private static object ParseValue(string type, string value, string format)
369+
{
370+
try
371+
{
372+
switch (type)
373+
{
374+
case "System.DateTime":
375+
DateTime.TryParseExact(value, format, new CultureInfo("en-US"), DateTimeStyles.None, out DateTime date);
376+
return date;
377+
default: throw new ArgumentException("DataType is not supported");
378+
}
379+
}
380+
catch(Exception e)
381+
{
382+
string text = String.Format("Type{0}, Value{1}, Format {2}", type, value, format);
383+
throw new ArgumentException(text, e);
384+
}
390385
}
391386
}
392387
}

0 commit comments

Comments
 (0)