Skip to content

Commit ab7304d

Browse files
rshewittrshewitt
committed
fix accessURL and downloadURL processing
1 parent 70d082b commit ab7304d

7 files changed

Lines changed: 85 additions & 67 deletions

File tree

‎lib/adiwg/mdtranslator/internal/module_utils.rb‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# frozen_string_literal: true
22

33
require 'net/http'
4+
require 'uri'
45

56
module AdiwgUtils
67
def self.reconcile_hashes(hashA, hashB)
@@ -134,4 +135,16 @@ def self.empty_string_to_nil(input)
134135

135136
input
136137
end
138+
139+
def self.normalized_path(url)
140+
# strips url query params and fragments for compatibility with File.extname
141+
# URI.parse("https://example.com/file.pdf?x=1#top").path -> /file.pdf
142+
# without it we have File.extname("https://example.com/file.pdf?download=1") -> .pdf?download=1
143+
uri = URI.parse(url)
144+
path = uri.path.to_s
145+
path = '/' if path.empty?
146+
path
147+
rescue URI::InvalidURIError
148+
''
149+
end
137150
end

‎lib/adiwg/mdtranslator/writers/dcat_us/sections/dcat_us_access_url.rb‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,26 @@
11
require 'jbuilder'
2+
require 'adiwg/mdtranslator/internal/module_utils'
3+
4+
PAGE_EXTENSIONS = Set.new(%w[
5+
.html .htm .xhtml .php .asp .aspx .jsp .cfm
6+
]).freeze
27

38
module ADIWG
4-
module Mdtranslator
5-
module Writers
6-
module Dcat_us
7-
module AccessURL
9+
module Mdtranslator
10+
module Writers
11+
module Dcat_us
12+
module AccessURL
13+
def self.access_url?(option)
14+
path = AdiwgUtils.normalized_path(option[:olResURI])
15+
ext = File.extname(path).downcase
816

9-
def self.build(option)
10-
option[:olResURI] if option[:olResURI].end_with?('.html')
11-
end
17+
# if the path ends with '/', or it doesn't have a file extension, or it has a common page extension
18+
return true if path.end_with?('/') || ext.empty? || PAGE_EXTENSIONS.include?(ext)
1219

13-
end
14-
end
20+
false
21+
end
22+
end
1523
end
16-
end
24+
end
25+
end
1726
end

‎lib/adiwg/mdtranslator/writers/dcat_us/sections/dcat_us_distribution.rb‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -61,24 +61,25 @@ def self.build(intObj)
6161
next unless option[:olResURI]
6262

6363
description = AdiwgUtils.empty_string_to_nil(option[:olResDesc])
64-
accessURL = AdiwgUtils.empty_string_to_nil(AccessURL.build(option))
65-
downloadURL = AdiwgUtils.empty_string_to_nil(DownloadURL.build(option))
64+
accessURL = AccessURL.access_url?(option)
65+
# if it's not a web page and it has a file extension then let's assume it's a downloadable file
66+
downloadURL = DownloadURL.download_url?(option) unless accessURL
6667

6768
# no point in creating a distribution if there's no link
68-
next if accessURL.nil? && downloadURL.nil?
69+
next unless accessURL || downloadURL
6970

70-
# we calculate the mediaType in the harvester
71-
# because it's unreliable in the source
72-
# mediaType is required if downloadURL is present
73-
mediaType = 'placeholder/value' if downloadURL
71+
# at this point one is true
72+
linkData = accessURL ? ['accessURL', option[:olResURI]] : ['downloadURL', option[:olResURI]]
73+
74+
# if there's a downloadURL there has to be a mediaType in dcatus 1.1 so using a placeholder
75+
mediaType = accessURL ? 'text/html' : 'placeholder/value'
7476

7577
title = AdiwgUtils.empty_string_to_nil(option[:olResName])
7678

7779
distribution = Jbuilder.new do |json|
7880
json.set!('@type', 'dcat:Distribution')
7981
json.set!('description', description)
80-
json.set!('accessURL', accessURL) if accessURL
81-
json.set!('downloadURL', downloadURL) if downloadURL
82+
json.set!(*linkData)
8283
json.set!('mediaType', mediaType)
8384
json.set!('title', title)
8485
end

‎lib/adiwg/mdtranslator/writers/dcat_us/sections/dcat_us_download_url.rb‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
require 'jbuilder'
2+
require 'adiwg/mdtranslator/internal/module_utils'
23

34
module ADIWG
4-
module Mdtranslator
5-
module Writers
6-
module Dcat_us
7-
module DownloadURL
8-
9-
def self.build(option)
10-
option[:olResURI] unless option[:olResURI].end_with?('.html')
11-
end
12-
13-
end
14-
end
5+
module Mdtranslator
6+
module Writers
7+
module Dcat_us
8+
module DownloadURL
9+
def self.download_url?(option)
10+
# avoid comparing the file extension to a limited set of values (e.g. .json, .xml, .rdf, etc...)
11+
path = AdiwgUtils.normalized_path(option[:olResURI])
12+
!File.extname(path).empty?
13+
end
14+
end
1515
end
16-
end
16+
end
17+
end
1718
end

‎test/translator/tc_iso19115_2_datagov_to_dcatus.rb‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -84,22 +84,20 @@ def test_distribution
8484

8585
res = dcatusNS.build(intMetadata)
8686

87-
expected = [
88-
{ '@type' => 'dcat:Distribution', 'description' => 'online resource description',
89-
'downloadURL' => 'online resource URL', 'title' => 'online resource name', 'mediaType' => 'placeholder/value' },
90-
{ '@type' => 'dcat:Distribution', 'description' => 'aggregate information detailed description',
91-
'downloadURL' => 'aggregate_information_online_resources',
92-
'title' => 'name of aggregate information resource', 'mediaType' => 'placeholder/value' },
93-
{ '@type' => 'dcat:Distribution', 'description' => 'aggregate information detailed description aoisd',
94-
'downloadURL' => 'aggregate_information_online_resources 12309u',
95-
'title' => 'name of aggregate information resource 10923j', 'mediaType' => 'placeholder/value' },
96-
{ '@type' => 'dcat:Distribution', 'description' => 'Aggregation Info Sample Description',
97-
'downloadURL' => 'https://aggregation_info_sample_url.gov',
98-
'title' => 'Aggregation Info Sample Name', 'mediaType' => 'placeholder/value' },
99-
{ '@type' => 'dcat:Distribution',
100-
'downloadURL' => 'https://online_resource_url.gov', 'title' => 'online resource name',
101-
'mediaType' => 'placeholder/value' }
102-
]
87+
expected = [{ '@type' => 'dcat:Distribution', 'description' => 'online resource description',
88+
'accessURL' => 'online resource URL', 'mediaType' => 'text/html', 'title' => 'online resource name' },
89+
{ '@type' => 'dcat:Distribution', 'description' => 'aggregate information detailed description',
90+
'accessURL' => 'aggregate_information_online_resources', 'mediaType' => 'text/html',
91+
'title' => 'name of aggregate information resource' },
92+
{ '@type' => 'dcat:Distribution', 'description' => 'aggregate information detailed description aoisd',
93+
'accessURL' => 'aggregate_information_online_resources 12309u', 'mediaType' => 'text/html',
94+
'title' => 'name of aggregate information resource 10923j' },
95+
{ '@type' => 'dcat:Distribution', 'description' => 'Aggregation Info Sample Description',
96+
'accessURL' => 'https://aggregation_info_sample_url.gov', 'mediaType' => 'text/html',
97+
'title' => 'Aggregation Info Sample Name' },
98+
{ '@type' => 'dcat:Distribution', 'accessURL' => 'https://online_resource_url.gov',
99+
'mediaType' => 'text/html',
100+
'title' => 'online resource name' }]
103101
assert_equal(expected, res)
104102
end
105103

‎test/translator/testData/iso19115-2-datagov-to-dcatus.json‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,35 +19,35 @@
1919
{
2020
"@type": "dcat:Distribution",
2121
"description": "online resource description",
22-
"downloadURL": "online resource URL",
23-
"mediaType": "placeholder/value",
22+
"accessURL": "online resource URL",
23+
"mediaType": "text/html",
2424
"title": "online resource name"
2525
},
2626
{
2727
"@type": "dcat:Distribution",
2828
"description": "aggregate information detailed description",
29-
"downloadURL": "aggregate_information_online_resources",
30-
"mediaType": "placeholder/value",
29+
"accessURL": "aggregate_information_online_resources",
30+
"mediaType": "text/html",
3131
"title": "name of aggregate information resource"
3232
},
3333
{
3434
"@type": "dcat:Distribution",
3535
"description": "aggregate information detailed description aoisd",
36-
"downloadURL": "aggregate_information_online_resources 12309u",
37-
"mediaType": "placeholder/value",
36+
"accessURL": "aggregate_information_online_resources 12309u",
37+
"mediaType": "text/html",
3838
"title": "name of aggregate information resource 10923j"
3939
},
4040
{
4141
"@type": "dcat:Distribution",
4242
"description": "Aggregation Info Sample Description",
43-
"downloadURL": "https://aggregation_info_sample_url.gov",
44-
"mediaType": "placeholder/value",
43+
"accessURL": "https://aggregation_info_sample_url.gov",
44+
"mediaType": "text/html",
4545
"title": "Aggregation Info Sample Name"
4646
},
4747
{
4848
"@type": "dcat:Distribution",
49-
"downloadURL": "https://online_resource_url.gov",
50-
"mediaType": "placeholder/value",
49+
"accessURL": "https://online_resource_url.gov",
50+
"mediaType": "text/html",
5151
"title": "online resource name"
5252
}
5353
],

‎test/writers/dcat_us/tc_dcat_us_distribution.rb‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,12 @@ def test_distribution
1616
hJsonOut = JSON.parse(metadata[:writerOutput])
1717
got = hJsonOut['distribution']
1818

19-
expect = [
20-
{ '@type' => 'dcat:Distribution', 'description' => 'distribution online resource description',
21-
'downloadURL' => 'http://ISO.uri/adiwg/0', 'mediaType' => 'placeholder/value' },
22-
{ '@type' => 'dcat:Distribution', 'downloadURL' => 'http://ISO.uri/adiwg/1',
23-
'mediaType' => 'placeholder/value' },
24-
{ '@type' => 'dcat:Distribution', 'description' => 'distribution description',
25-
'downloadURL' => 'http://ISO.uri/adiwg/3', 'mediaType' => 'placeholder/value' },
26-
{ '@type' => 'dcat:Distribution', 'downloadURL' => 'http://ISO.uri/adiwg/2',
27-
'mediaType' => 'placeholder/value' }
28-
]
19+
expect = [{ '@type' => 'dcat:Distribution', 'description' => 'distribution online resource description',
20+
'accessURL' => 'http://ISO.uri/adiwg/0', 'mediaType' => 'text/html' },
21+
{ '@type' => 'dcat:Distribution', 'accessURL' => 'http://ISO.uri/adiwg/1', 'mediaType' => 'text/html' },
22+
{ '@type' => 'dcat:Distribution', 'description' => 'distribution description', 'accessURL' => 'http://ISO.uri/adiwg/3',
23+
'mediaType' => 'text/html' },
24+
{ '@type' => 'dcat:Distribution', 'accessURL' => 'http://ISO.uri/adiwg/2', 'mediaType' => 'text/html' }]
2925

3026
assert_equal expect, got
3127
end

0 commit comments

Comments
 (0)