detect broken links using wget and sitemap

Author: starswan

.github/actions/check-broken-links/action.yml

@@ -0,0 +1,38 @@
+name: check-broken-links action
+description: check broken links application
+
+inputs:
+  aks_environment:
+    description: Environment
+    required: true
+  event_name:
+    description: type of event that triggred the test
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Checkout code
+      uses: actions/checkout@v6
+
+    - name: Prepare application environment
+      uses: ./.github/actions/prepare-app-env
+
+    - name: set environment (scheduled test)
+      shell: bash
+      if: ${{ inputs.event_name == 'schedule' }}
+      run: echo "AKS_ENVIRONMENT=production" >> $GITHUB_ENV
+
+    - name: set environment
+      shell: bash
+      if: ${{ inputs.event_name != 'schedule' }}
+      run: |
+        echo "AKS_ENVIRONMENT=${{ inputs.aks_environment }}" >> $GITHUB_ENV
+
+    - name: Run deployment link check
+      shell: bash
+      run:  ./check_broken_links.sh "https://teaching-vacancies-${{ inputs.aks_environment }}.test.teacherservices.cloud" "${{ inputs.http_basic_user }}" "${{ inputs.http_basic_password }}"
+
+    - name: print environment
+      shell: bash
+      run: echo ${{ env.AKS_ENVIRONMENT}}

.github/workflows/build_and_deploy.yml

@@ -317,6 +317,15 @@ jobs:
       if: steps.smoke-test.conclusion != 'success'
       run: exit 1
 
+    - name: Trigger Broken Link action
+      id: check-broken-links
+      uses: ./.github/actions/check-broken-links/
+      with:
+        aks_environment: ${{ env.ENVIRONMENT }}
+        event_name: ${{ github.event_name }}
+        http_basic_user: ${{ secrets.HTTP_BASIC_USER }}
+        http_basic_password: ${{ secrets.HTTP_BASIC_PASSWORD }}
+
     - name: Post sticky pull request comment
       if: github.event_name == 'pull_request'
       uses: marocchino/sticky-pull-request-comment@v2

.simplecov

@@ -32,7 +32,6 @@ if ENV.fetch("COVERAGE", 0).to_i.positive?
 
     # Filters out files from coverage reports
     add_filter "app/services/custom_log_formatter.rb"
-    add_filter "app/controllers/robots_controller.rb"
     add_filter "app/controllers/previews_controller.rb"
     add_filter "app/controllers/sha_controller.rb"
     add_filter "app/jobs/set_organisation_slugs_job.rb"

app/controllers/sitemap_controller.rb

@@ -1,5 +1,8 @@
 class SitemapController < ApplicationController
-  def show # rubocop:disable Metrics/AbcSize
+  STATIC_PAGES = %w[terms-and-conditions savings-methodology accessibility vision-statement].freeze
+  POST_SECTION_NAMES = %w[get-help-hiring jobseeker-guides].freeze
+
+  def show # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
     map = XmlSitemap::Map.new(service_domain, secure: !Rails.env.development?) do |m|
       # Live vacancies
       PublishedVacancy.live.applicable.find_each do |vacancy|
@@ -16,10 +19,29 @@ def show # rubocop:disable Metrics/AbcSize
         m.add location_landing_page_path(location.parameterize), period: "hourly"
       end
 
-      # Static pages
-      m.add page_path("terms-and-conditions"), period: "weekly"
-      m.add page_path("cookies"), period: "weekly"
-      m.add page_path("accessibility"), period: "weekly"
+      STATIC_PAGES.each { |static_page| m.add page_path(static_page), period: "weekly" }
+
+      POST_SECTION_NAMES.each do |section|
+        m.add posts_path(section), period: "weekly"
+
+        MarkdownDocument.all_subcategories(section).each do |sub_category|
+          m.add subcategory_path(section, sub_category.post_name), period: "weekly"
+          posts = MarkdownDocument.all(section, sub_category.post_name)
+          posts.each do |post|
+            m.add post_path(section, sub_category.post_name, post.post_name), period: "weekly"
+          end
+        end
+      end
+      # POST_SECTIONS.each do |section, subcats|
+      #   m.add posts_path(section), period: "weekly"
+      #   subcats.each do |subcat, post|
+      #     m.add subcategory_path(section, subcat), period: "weekly"
+      #
+      #     post.each do |post|
+      #       m.add post_path(section, subcat, post), period: "weekly"
+      #     end
+      #   end
+      # end
     end
 
     expires_in 3.hours

app/views/robots/show.text.erb

@@ -1,5 +1,5 @@
-<%- if Rails.configuration.app_role.production? %>
-SITEMAP: https://teaching-vacancies.service.gov.uk/sitemap.xml
+<%- if Rails.configuration.app_role.production? || Rails.configuration.app_role.review? %>
+SITEMAP: /sitemap.xml
 User-agent: *
 Allow: /
 Disallow: /check

check_broken_links.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+
+# parameters 1 - base URL 2 - user - 3 password
+
+# grab sitemap and roughly parse it into a link of pages to be passed to wget spider-mode
+# Try not to spider school sites or big slow govuk ones.
+
+# get-information-schools.service.gov.uk refuses to be spidered so we
+# can't check broken links to their domain
+# signin.education.co.uk and friends don't respond to robots.txt
+#
+wget --auth-no-challenge -q --user=$2 --password=$3 $1/sitemap.xml -O - \
+  | fgrep loc \
+  | sed s'/    <loc>//' \
+  | sed s'/<\/loc>//' \
+  | wget -nv -np -w 0.1 --spider -H -r -l1 -i - --user=$2 --password=$3 -P /tmp/spider \
+  --auth-no-challenge \
+  --no-relative \
+  --domains="gov.uk" \
+  --exclude-domains="signin.education.gov.uk,get-information-schools.service.gov.uk,ofsted.gov.uk,nationalarchives.gov.uk"

config/routes.rb

@@ -33,7 +33,7 @@
   get "/get-help-hiring/accepting-job-applications-on-teaching-vacancies", to: redirect { |_params, _request|
     Rails.application.routes.url_helpers.post_path(section: "get-help-hiring", subcategory: "how-to-create-job-listings-and-accept-applications", post_name: "accepting-job-applications-on-teaching-vacancies")
   }
-  get "/get-help-hiring/communicating-with-jobskeers", to: redirect { |_params, _request|
+  get "/get-help-hiring/communicating-with-jobseekers", to: redirect { |_params, _request|
     Rails.application.routes.url_helpers.post_path(section: "get-help-hiring", subcategory: "how-to-create-job-listings-and-accept-applications", post_name: "communicating-with-jobseekers")
   }
   get "/jobseeker-guides/write-a-great-teaching-job-application-in-five-steps", to: redirect { |_params, _request|

spec/requests/sitemap_spec.rb

@@ -40,10 +40,13 @@
 
     it "includes static pages in the sitemap" do
       expect(xml).to include(page_path("terms-and-conditions"))
-      expect(xml).to include(page_path("cookies"))
       expect(xml).to include(page_path("accessibility"))
     end
 
+    it "includes posts in the sitemap" do
+      expect(xml).to include(post_path("get-help-hiring", "how-to-create-job-listings-and-accept-applications", "creating-the-perfect-teacher-job-advert"))
+    end
+
     it "sets cache expiry" do
       expect(response.headers["Cache-Control"]).to include("max-age=10800")
     end