@@ -66,19 +66,10 @@ type APIRequest struct {
6666 URL string `json:"url"`
6767 URLs []string `json:"urls"`
6868
69- AllowDomains []string `json:"domains"`
70- AllowPaths []string `json:"paths"`
71- DenyPaths []string `json:"!paths"`
69+ AliasedDomains []string `json:"aliases"`
70+ IgnorePaths []string `json:"ignores"`
7271
73- UserAgent string `json:"user_agent"`
74-
75- // If true, we'll bypass the robots.txt check, however we'll still
76- // download the file to look for sitemaps.
77- SkipRobots bool `json:"skip_robots"`
78-
79- // If true we'll not use any sitemaps found automatically, only those that
80- // have been explicitly provided.
81- SkipSitemapDiscovery bool `json:"skip_sitemap_discovery"`
72+ UserAgent string `json:"ua"`
8273
8374 // A list of authentication configurations, that are used in the run.
8475 AuthConfigs []* AuthConfig `json:"auth"`
@@ -126,37 +117,33 @@ func (req *APIRequest) GetURLs(clean bool) []string {
126117}
127118
128119func (req * APIRequest ) GetAllowDomains () []string {
129- // Ensure at least the URL host is in allowed domains.
130120 var domains []string
131- if req .AllowDomains != nil {
132- domains = req .AllowDomains
133- } else {
134- for _ , u := range req .GetURLs (false ) {
135- p , err := url .Parse (u )
136- if err != nil {
137- slog .Error ("Failed to parse URL from request, not allowing that domain." , "url" , u , "error" , err )
138- continue
139- }
140- domains = append (domains , p .Hostname ())
121+
122+ // The domains of the targets are allways allowed.
123+ for _ , u := range req .GetURLs (false ) {
124+ p , err := url .Parse (u )
125+ if err != nil {
126+ slog .Error ("Failed to parse URL from request, not allowing that domain." , "url" , u , "error" , err )
127+ continue
141128 }
129+ domains = append (domains , p .Hostname ())
142130 }
143- return domains
144- }
145131
146- func (req * APIRequest ) GetAllowPaths () []string {
147- var paths []string
148-
149- if req .AllowPaths != nil {
150- paths = req .AllowPaths
132+ if req .AliasedDomains != nil {
133+ domains = append (domains , req .AliasedDomains ... )
151134 }
152- return paths
135+
136+ return domains
153137}
154138
155- func (req * APIRequest ) GetDenyPaths () []string {
139+ func (req * APIRequest ) GetIgnorePaths () []string {
156140 var paths []string
157141
158- if req .DenyPaths != nil {
159- paths = req .DenyPaths
142+ if req .IgnorePaths == nil {
143+ return paths
144+ }
145+ for _ , p := range req .IgnorePaths {
146+ paths = append (paths , p )
160147 }
161148 return paths
162149}
@@ -228,14 +215,3 @@ type APIResponse struct {
228215type APIError struct {
229216 Message string `json:"message"`
230217}
231-
232- type CloudEventJson struct {
233- Specversion string `json:"specversion"`
234- Event_type string `json:"type"`
235- Source string `json:"source"`
236- Subject string `json:"subject"`
237- Id string `json:"id"`
238- Time string `json:"time"`
239- Datacontenttype string `json:"datacontenttype"`
240- Data string `json:"data"`
241- }
0 commit comments