Skip to content

Commit 6512d8d

Browse files
kpuriIpackpuriIpac
committed
FIREFLY-1927: Refactor obscore packager
1 parent 50f19f4 commit 6512d8d

File tree

9 files changed

+914
-638
lines changed

9 files changed

+914
-638
lines changed

‎src/firefly/java/edu/caltech/ipac/firefly/server/packagedata/obscorepackager/DatalinkUtil.java‎

Lines changed: 354 additions & 0 deletions
Large diffs are not rendered by default.

‎src/firefly/java/edu/caltech/ipac/firefly/server/packagedata/obscorepackager/ObsCorePackager.java‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
package edu.caltech.ipac.firefly.server.packagedata.obscorepackager;
2+
3+
import edu.caltech.ipac.firefly.data.DownloadRequest;
4+
import edu.caltech.ipac.firefly.data.FileInfo;
5+
import edu.caltech.ipac.firefly.data.ServerRequest;
6+
import edu.caltech.ipac.firefly.server.db.EmbeddedDbUtil;
7+
import edu.caltech.ipac.firefly.server.network.HttpServiceInput;
8+
import edu.caltech.ipac.firefly.server.packagedata.FileGroup;
9+
import edu.caltech.ipac.firefly.server.query.DataAccessException;
10+
import edu.caltech.ipac.firefly.server.query.FileGroupsProcessor;
11+
import edu.caltech.ipac.firefly.server.query.SearchProcessorImpl;
12+
import edu.caltech.ipac.firefly.server.util.Logger;
13+
import edu.caltech.ipac.table.DataGroup;
14+
import edu.caltech.ipac.table.MappedData;
15+
16+
import java.io.IOException;
17+
import java.net.URI;
18+
import java.net.URL;
19+
import java.util.*;
20+
21+
import static edu.caltech.ipac.firefly.server.db.EmbeddedDbUtil.getSelectedData;
22+
import static edu.caltech.ipac.firefly.server.packagedata.obscorepackager.DatalinkUtil.*;
23+
import static edu.caltech.ipac.firefly.server.packagedata.obscorepackager.ObsCoreUtil.*;
24+
25+
@SearchProcessorImpl(id = "ObsCorePackager")
26+
public class ObsCorePackager extends FileGroupsProcessor {
27+
private static final Logger.LoggerImpl LOGGER = Logger.getLogger();
28+
public static final List<String> acceptableExtensionTypes = Arrays.asList("png","jpg","jpeg","bmp","fits","tsv",
29+
"csv","tbl","json","pdf","tar","html","xml","vot","reg");
30+
public static final String PRODUCTS = "productTypes";
31+
public static final String ACCESS_URL = "access_url";
32+
public static final String SERVICE_DEF = "service_def";
33+
public static final String ACCESS_FORMAT = "access_format";
34+
public static final String SEMANTICS = "semantics";
35+
public static final String CONTENT_TYPE = "content_type";
36+
public static final String FILE = "file";
37+
public static final String SOURCE = "source";
38+
public static final String DATALINK_SER_DEF = "datalinkServiceDescriptor";
39+
public static final String ADHOC_SERVICE = "adhoc:service";
40+
public static final String DATALINK = "datalink";
41+
public static final String CENTER_COL_NAMES = "centerColNames";
42+
public static final String CENTER_COL_VALS = "centerColValues";
43+
public static final String LON_COL = "lonCol";
44+
public static final String LAT_COL = "latCol";
45+
public static final String RA = "ra";
46+
public static final String DEC = "dec";
47+
public static final String CUTOUT_VALUE = "cutoutValue";
48+
public static final String SER_DEF_ACCESS_URL = "accessURL";
49+
public static final String SER_DEF_INPUT_PARAMS = "inputParams";
50+
public static final String CIRCLE = "circle";
51+
public static final String REF = "ref";
52+
public static final String VALUE = "value";
53+
public static final String TEMPLATE_COL_NAMES = "templateColNames";
54+
public static final String USE_SOURCE_FILE_NAME = "useSourceUrlFileName";
55+
public static final String POSITION = "position";
56+
public static final String GENERATE_DOWNLOAD_FILE_NAME = "generateDownloadFileName";
57+
58+
public static final List<String> CUTOUT_UCDs= Arrays.asList("phys.size","phys.size.radius","phys.angSize", "pos.spherical.r");
59+
public static final List<String> RA_UCDs= List.of("pos.eq.ra");
60+
public static final List<String> DEC_UCDs= List.of("pos.eq.dec");
61+
62+
public static final List<String> DATALINK_COL_NAMES = List.of(
63+
"id", "access_url", "service_def", "error_message", "semantics",
64+
"description", "content_type", "content_length"
65+
);
66+
67+
@Override
68+
public List<FileGroup> loadData(ServerRequest request) throws IOException, DataAccessException {
69+
try {
70+
return computeFileGroup((DownloadRequest) request);
71+
} catch (Exception e) {
72+
LOGGER.error(e);
73+
throw e;
74+
}
75+
}
76+
77+
private List<FileGroup> computeFileGroup(DownloadRequest request) throws DataAccessException {
78+
List<FileInfo> fileInfos = new ArrayList<>();
79+
try {
80+
var selectedRows = new ArrayList<>(request.getSelectedRows());
81+
MappedData dgDataUrl = EmbeddedDbUtil.getSelectedMappedData(request.getSearchRequest(), selectedRows); //returns all columns
82+
DataGroup dg = getSelectedData(request.getSearchRequest(), selectedRows);
83+
84+
String pos = request.getParam(POSITION);
85+
Map<String, Integer> prefixCounter = new HashMap<>();
86+
87+
//todo: remove datalinkServDesc from here, moved it down where it's in scope
88+
//String datalinkServDesc = request.getParam(DATALINK_SER_DEF); //check for a non-obscore table containing a Datalink Service Descriptor
89+
90+
//todo: remove this: DataType[] dataDefs = dg.getDataDefinitions();
91+
boolean isLinksTable = isLinksTable(dg); //todo: remove this: isLinksTable(dataDefs);
92+
93+
if (isLinksTable) { //to check if the incoming request is directly from a datalink table (in cases where user may extract a table from an existing obscore type table)
94+
List<String> positionColVals = getPositionColVals(pos, -1, null); //don't need index or dgDataUrl, pos should have ra/dec vals
95+
List<FileInfo> tmpFileInfos = resolveDatalink(null, request, "", positionColVals, dg);
96+
fileInfos.addAll(tmpFileInfos);
97+
}
98+
else {
99+
for (int idx : selectedRows) {
100+
String accessUrl = (String) dgDataUrl.get(idx, ACCESS_URL);
101+
//check for a non-obscore table containing a Datalink Service Descriptor
102+
String datalinkServDesc = request.getParam(DATALINK_SER_DEF); //note: datalinkServDesc is a JSON string
103+
104+
if (accessUrl == null) {
105+
//check if this could be a non-obscore table containing a datalink url
106+
if (datalinkServDesc != null) {
107+
//construct product url / access url here
108+
DatalinkUtil.ServDescUrl serDescUrl = createUrlFromServDesc(datalinkServDesc);
109+
accessUrl = getAccessUrlFromNonObsCore(serDescUrl, dg, dgDataUrl, idx);
110+
} else continue; //no file available to process
111+
}
112+
113+
String accessFormat = (String) dgDataUrl.get(idx, ACCESS_FORMAT);
114+
URL url = new URI(accessUrl).toURL();
115+
116+
List<String> positionColVals = getPositionColVals(pos, idx, dgDataUrl);
117+
String colNames = request.getSearchRequest().getParam(TEMPLATE_COL_NAMES);
118+
String[] cols = colNames != null ? colNames.split(",") : null;
119+
String fileNamePrefix = getFileNamePrefix(idx, cols, dgDataUrl, url, positionColVals); //this will be used as folder name, and as prefix for individual file names
120+
fileNamePrefix = makeUniquePrefix(fileNamePrefix, prefixCounter);
121+
122+
boolean isTblContainsDatalink = (datalinkServDesc != null) || (accessFormat != null && accessFormat.contains(DATALINK));
123+
124+
if (isTblContainsDatalink) {
125+
List<FileInfo> tmpFileInfos = resolveDatalink(url, request, fileNamePrefix, positionColVals, null);
126+
fileInfos.addAll(tmpFileInfos);
127+
} else {
128+
String extName = null;
129+
FileInfo fileInfo = new FileInfo(accessUrl, extName, 0);
130+
fileInfos.add(fileInfo);
131+
}
132+
}
133+
}
134+
fileInfos.forEach(fileInfo -> fileInfo.setRequestInfo(new HttpServiceInput(fileInfo.getInternalFilename())));
135+
}
136+
catch(Exception e) {
137+
LOGGER.error(e);
138+
throw new DataAccessException("Error while processing ObsCorePackager request: " + e.getMessage(), e);
139+
}
140+
return List.of(new FileGroup(fileInfos, null, 0, "ObsCore Download Files"));
141+
}
142+
}

‎src/firefly/java/edu/caltech/ipac/firefly/server/packagedata/obscorepackager/ObsCoreUtil.java‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package edu.caltech.ipac.firefly.server.packagedata.obscorepackager;
2+
3+
import edu.caltech.ipac.firefly.server.util.Logger;
4+
import edu.caltech.ipac.table.MappedData;
5+
import org.json.JSONObject;
6+
7+
import java.net.URL;
8+
import java.util.List;
9+
import java.util.Map;
10+
import java.util.Objects;
11+
12+
import static edu.caltech.ipac.firefly.core.Util.Opt.ifNotNull;
13+
import static org.apache.commons.lang.StringUtils.isEmpty;
14+
15+
public class ObsCoreUtil {
16+
17+
private static final Logger.LoggerImpl LOGGER = Logger.getLogger();
18+
19+
public record CenterCols(String lonCol, String latCol) {}
20+
public record CenterColValues(String ra, String dec) {}
21+
public record Position(CenterCols centerColNames, CenterColValues centerColValues) {}
22+
23+
public static boolean testSem(String sem, String val) {
24+
return (sem != null && sem.toLowerCase().endsWith(val));
25+
}
26+
27+
public static String getExtFromURL(String contentTypeOrUrl) {
28+
String ext = "unknown";
29+
try {
30+
if (contentTypeOrUrl != null) {
31+
for (String ct : ObsCorePackager.acceptableExtensionTypes) {
32+
if (contentTypeOrUrl.contains(ct)) {
33+
ext = ct;
34+
break;
35+
}
36+
}
37+
}
38+
} catch (Exception e) {
39+
LOGGER.error(e);
40+
}
41+
return ext;
42+
}
43+
44+
public static List<String> getPositionColVals(String pos, int idx, MappedData dgDataUrl) {
45+
Position position = getPosition(pos);
46+
String ra, dec;
47+
48+
ra = ifNotNull(position.centerColValues().ra())
49+
.get(v -> v.equals("null") ? null : v);
50+
51+
dec = ifNotNull(position.centerColValues().dec())
52+
.get(v -> v.equals("null") ? null : v);
53+
54+
if (ra == null || dec == null) {
55+
String lonCol = position.centerColNames().lonCol();
56+
String latCol = position.centerColNames().latCol();
57+
ra = Objects.toString(dgDataUrl.get(idx, lonCol), null);
58+
dec = Objects.toString(dgDataUrl.get(idx, latCol), null);
59+
}
60+
return List.of(ra, dec);
61+
}
62+
63+
private static Position getPosition(String pos) {
64+
JSONObject jsonObject = new JSONObject(pos);
65+
66+
String lonCol = ObsCorePackager.RA;
67+
String latCol = ObsCorePackager.DEC;
68+
69+
if (jsonObject.has(ObsCorePackager.CENTER_COL_NAMES)) {
70+
JSONObject centerColsJson = jsonObject.getJSONObject(ObsCorePackager.CENTER_COL_NAMES);
71+
if (centerColsJson.has(ObsCorePackager.LON_COL)) {
72+
lonCol = String.valueOf(centerColsJson.get(ObsCorePackager.LON_COL));
73+
}
74+
if (centerColsJson.has(ObsCorePackager.LAT_COL)) {
75+
latCol = String.valueOf(centerColsJson.get(ObsCorePackager.LAT_COL));
76+
}
77+
}
78+
79+
CenterCols centerColNames = new CenterCols(lonCol, latCol);
80+
81+
JSONObject centerColValuesJson = jsonObject.getJSONObject(ObsCorePackager.CENTER_COL_VALS);
82+
String raVal = String.valueOf(centerColValuesJson.opt(ObsCorePackager.RA));
83+
String decVal = String.valueOf(centerColValuesJson.opt(ObsCorePackager.DEC));
84+
85+
CenterColValues centerColValues = new CenterColValues(raVal, decVal);
86+
87+
return new Position(centerColNames, centerColValues);
88+
}
89+
90+
public static String makeUniquePrefix(String basePrefix, Map<String, Integer> prefixCounter) {
91+
if (!prefixCounter.containsKey(basePrefix)) {
92+
prefixCounter.put(basePrefix, 1);
93+
return basePrefix;
94+
} else {
95+
int count = prefixCounter.get(basePrefix);
96+
String newPrefix = basePrefix + "-" + count;
97+
while (prefixCounter.containsKey(newPrefix)) {
98+
count++;
99+
newPrefix = basePrefix + "-" + count;
100+
}
101+
prefixCounter.put(basePrefix, count);
102+
prefixCounter.put(newPrefix, 1);
103+
return newPrefix;
104+
}
105+
}
106+
107+
public static String makeValidString(String val) {
108+
return val.replaceAll("\\s", "").replaceAll("[^a-zA-Z0-9._-]", "_");
109+
}
110+
111+
public static String getFileNamePrefix(int idx, String[] colNames, MappedData dgDataUrl, URL url, List<String> positionColVals) {
112+
String obsCollection = (String) dgDataUrl.get(idx, "obs_collection");
113+
String instrumentName = (String) dgDataUrl.get(idx, "instrument_name");
114+
String obsId = (String) dgDataUrl.get(idx, "obs_id");
115+
String obsTitle = (String) dgDataUrl.get(idx, "obs_title");
116+
String ra = positionColVals.get(0);
117+
String dec = positionColVals.get(1);
118+
119+
String file_name = "";
120+
121+
if (colNames != null) {
122+
for (String col : colNames) {
123+
String value = (String) dgDataUrl.get(idx, col);
124+
if (value != null) {
125+
file_name += makeValidString(value) + "-";
126+
}
127+
}
128+
}
129+
if (file_name.length() > 0) {
130+
return file_name.substring(0, file_name.length() - 1);
131+
} else if (obsTitle != null) {
132+
file_name = makeValidString(obsTitle);
133+
} else {
134+
file_name = (obsCollection != null ? makeValidString(obsCollection) + "-" : "") +
135+
(instrumentName != null ? makeValidString(instrumentName) + "-" : "") +
136+
(obsId != null ? makeValidString(obsId) : "");
137+
}
138+
139+
if (file_name.length() == 0) {
140+
if (!isEmpty(ra) && !isEmpty(dec)) {
141+
file_name = ra + "_" + dec;
142+
} else {
143+
file_name = "file_" + idx;
144+
}
145+
}
146+
return file_name;
147+
}
148+
}

0 commit comments

Comments
 (0)