diff --git a/README.md b/README.md index 5f8a320..cc98965 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ Provides additional configuration, styling and components for the Drupal Webform * LocalGov Forms Date - A date input field based on the [GDS Date Input pattern](https://design-system.service.gov.uk/components/date-input/) * LocalGov address lookup - Webform element with a configurable address lookup backend. Geocoder plugins act as backends. +## Plugins +- Personally Identifiable Information (PII) redactor from Webform submissions: At the moment, a plugin manager `plugin.manager.pii_redactor` and a sample plugin are provided. + ## Dependencies The geocoder-php/nominatim-provider package is necessary to run automated tests: ``` @@ -23,4 +26,3 @@ To avoid the configuration being removed by deployments, install the [Config ign webform.webform.* webform.webform_options.* ``` - diff --git a/js/address_change.js b/js/address_change.js index dbc72e0..51b2eb2 100644 --- a/js/address_change.js +++ b/js/address_change.js @@ -58,6 +58,7 @@ * Populate Central Hub Selected Address * * Adds the selected address to drupalSettings.centralhub.selectedAddress. + * * @param {jQuery} selectList * Address selectlist element. */ @@ -87,7 +88,7 @@ var searchButton = addressLookupElement.find('.js-address-searchbutton'); var selectListContainer = addressLookupElement.find('.js-address-select-container'); var selectList = selectListContainer.find('.js-address-select'); - var error = selectListContainer.find('.js-address-error'); + var error = selectListContainer.find('.js-address-error, .error'); // Change the search button to normal button. searchButton.attr('type', 'button'); @@ -154,7 +155,7 @@ var searchButton = addressLookupElement.find('.js-address-searchbutton'); var selectListContainer = addressLookupElement.find('.js-address-select-container'); var selectList = selectListContainer.find('.js-address-select'); - var error = selectListContainer.find('.js-address-error'); + var error = selectListContainer.find('.js-address-error, .error'); var resetButton = addressLookupElement.find('.js-reset-address'); var manualButton = addressLookupElement.find('.js-manual-address'); var manualAddressContainer = addressLookupElement.find('+ .js-address-entry-container'); diff --git a/js/address_select.js b/js/address_select.js index 0498c72..e51513d 100644 --- a/js/address_select.js +++ b/js/address_select.js @@ -44,7 +44,6 @@ function hideManualAddress(centralHubElement, type, settings) { var manualAddressContainer = centralHubElement.find('.js-address-entry-container'); var manualButton = centralHubElement.find('.js-manual-address'); - var addressSelectContainer = centralHubElement.find('.js-address-select-container'); manualAddressContainer.addClass('hidden'); if (type == 'hard') { @@ -63,6 +62,7 @@ /** * Show the manual address form elements. + * * @param {jQuery} centralHubElement * Centralhub address element. */ @@ -70,17 +70,10 @@ var manualAddressContainer = centralHubElement.find('.js-address-entry-container'); var manualButton = centralHubElement.find('.js-manual-address'); var addressSelectContainer = centralHubElement.find('.js-address-select-container'); - var addressSelect = addressSelectContainer.find('select'); var searchElement = centralHubElement.find('.js-address-searchstring'); var addressError = addressSelectContainer.find('.js-address-error'); manualAddressContainer.removeClass('hidden'); - // manualAddressContainer.find('input').val(''); manualButton.hide(); - // addressSelectContainer.addClass('hidden'); - // addressSelect.val('0'); - // Clear the search element when entering a manual address. - // This is to pass validation. - // searchElement.val(''); // Remove the error element when making a manual address. addressError.remove(); } @@ -174,7 +167,6 @@ central_hub_webform_address_container.find('input.js-localgov-forms-webform-uk-address--' + value).val(addressSelected[value]); }); - // hideManualAddress(centralHubElement, 'soft'); showManualAddress(centralHubElement); } else if ($(this).val() == 0) { // If choosing the empty option, clear out the address fields. @@ -211,9 +203,6 @@ hideManualAddress(centralHubElement, 'soft', settings); } - // centralHubElement.find('.js-address-searchstring').change(function() { - // hideManualAddress(centralHubElement, 'hard'); - // }); centralHubElement.find('.js-reset-address').click(function () { hideManualAddress(centralHubElement, 'hard', settings); }); diff --git a/localgov_forms.services.yml b/localgov_forms.services.yml index 233229d..959c86b 100644 --- a/localgov_forms.services.yml +++ b/localgov_forms.services.yml @@ -6,3 +6,8 @@ services: localgov_forms.address_lookup: class: Drupal\localgov_forms\AddressLookup arguments: ['@geocoder', '@localgov_forms.geocoder_selection'] + + # Plugin manager service for PII redaction from Webform submissions. + plugin.manager.pii_redactor: + class: Drupal\localgov_forms\Plugin\PIIRedactorPluginManager + parent: default_plugin_manager diff --git a/modules/localgov_forms_lts/README.md b/modules/localgov_forms_lts/README.md new file mode 100644 index 0000000..b0ae34b --- /dev/null +++ b/modules/localgov_forms_lts/README.md @@ -0,0 +1,108 @@ +## Long term storage for Webform submissions +This module copies Webform submissions to a separate database for Long Term Storage (LTS). The LTS database can then be used for data warehousing needs such as reporting and analysis. Optionally, Personally Identifiable Information (PII) can be redacted from Webform submissions while they are copied to the LTS database. + +### Setup process +- Create a database which will serve as the LTS database. +- Declare it in Drupal's settings.php using the **localgov_forms_lts** key. Example: + ``` + $databases['localgov_forms_lts']['default'] = [ + 'database' => 'our-long-term-storage-database', + 'username' => 'database-username-goes-here', + 'password' => 'database-password-goes-here', + 'host' => 'database-hostname-goes-here', + 'port' => '3306', + 'driver' => 'mysql', + 'prefix' => '', + ]; + ``` +- Install the localgov_forms_lts submodule. +- Check the module requirement report from Drupal's status page at `admin/reports/status`. This should be under the **LocalGov Forms LTS** key. +- [Optional] If all looks good in the previous step, run `drush localgov-forms-lts:copy --force` which will copy existing Webform submissions into the LTS database. +- By default, periodic Webform submissions copying to the LTS database is disabled. Activate it from `/admin/structure/webform/config/submissions-lts`. +- Ensure cron is running periodically. This will copy any new Webform submissions or changes to existing Webform submissions since the last cron run or the last `drush localgov-forms-lts:copy` run. +- [Optional] Tell individual Webforms to purge submissions older than a chosen period. This is configured for each Webform from its `Settings > Submissions > Submission purge settings` configuration section. + +### Inspection +To inspect Webform submissions kept in Long term storage, look for the **LTS** tab in the Webform submissions listing page. This is usually at `/admin/structure/webform/submissions/manage`. + +### Good to know +- Each cron run copies 50 Webform submissions. If your site is getting more than that many Webform submissions between subsequent cron runs, not all Webform submissions will get copied to LTS during a certain period. If that happens, adjust cron run frequency. +- Files attached to Webform submissions are *not* moved to LTS. +- You can choose to redact elements with Personally Identifiable Information (PII) while they are copied to the LTS database. For that, select *Best effort PII redactor* (or another redactor) from the `PII redactor plugin` dropdown in the LTS config page at `/admin/structure/webform/config/submissions-lts`. At the moment, this plugin redacts all name, email, telephone, number, and various address type elements. Additionally, any text or radio or checkbox element whose machine name (AKA Key) contains the following also gets redacted: name, mail, phone, contact_number, date_of_birth, dob_, personal_, title, nino, passport, postcode, address, serial_number, reg_number, pcn_, and driver_. +- If you are using this module in multiple instances of the same site (e.g. dev/stage/live), ensure that the database settings array points to *different* databases. Alternatively, disable copying for the non-live environments from `/admin/structure/webform/config/submissions-lts`. The relevant settings `localgov_forms_lts.settings:is_copying_enabled` can be [overridden](https://www.drupal.org/docs/drupal-apis/configuration-api/configuration-override-system#s-global-overrides) from settings.php. The [config_split](https://www.drupal.org/project/config_split) module can be handy as well. +- This module is currently in experimental stage. + +### Todo +- Removal of Webform submissions from LTS after a predefined period e.g. 5 years. + +### Testing in DDEV + +To set up testing in ddev, we'll need to set up a second database. + +There are a few ways to do this, but the following seems to work. + +#### 1. Add a post-start hook to your .ddev/config.yml + +Edit `.ddev/config.yml` and add the following to create a new database on start. + +``` +hooks: + post-start: + - exec: mysql -uroot -proot -e "CREATE DATABASE IF NOT EXISTS localgov_forms_lts; GRANT ALL ON localgov_forms_lts.* to 'db'@'%';" + service: db +``` + +#### 2. Add the database connection string to settings.php: + +Edit sites/default/settings.php and add a new database connection string at the +end of the file. + +``` +// Database connection for localgov_forms_lts. +$databases['localgov_forms_lts']['default'] = [ + 'database' => 'localgov_forms_lts', + 'username' => 'db', + 'password' => 'db', + 'host' => 'db', + 'port' => '3306', + 'driver' => 'mysql', + 'prefix' => '', +]; +``` + +#### 3. Install Adminer + +Adminer is useful if you want to inspect databases and tables. + +``` +ddev get ddev/ddev-adminer +``` + +#### 4. Restart ddev + +``` +ddev restart +``` + +#### 5. Require and install the module. + +``` +ddev composer require localgovdrupal/localgov_forms +ddev drush si localgov_forms_lts -y +``` + +#### 5. Make some submissions. + +For example, in LocalGov Drupal we tend to have a contact form at /form/contact. + +Make a couple of submissions there. + +#### 6. Run cron. + +ddev drush cron + +#### 7. Inspect the LTS tab + +Go to /admin/structure/webform/submissions/lts + +Here you should see your submissions with redacted name and email address. diff --git a/modules/localgov_forms_lts/config/install/localgov_forms_lts.settings.yml b/modules/localgov_forms_lts/config/install/localgov_forms_lts.settings.yml new file mode 100644 index 0000000..09fcfc7 --- /dev/null +++ b/modules/localgov_forms_lts/config/install/localgov_forms_lts.settings.yml @@ -0,0 +1,2 @@ +is_copying_enabled: false +pii_redactor_plugin_id: '' diff --git a/modules/localgov_forms_lts/config/schema/localgov_forms_lts.schema.yml b/modules/localgov_forms_lts/config/schema/localgov_forms_lts.schema.yml new file mode 100644 index 0000000..b4af9d6 --- /dev/null +++ b/modules/localgov_forms_lts/config/schema/localgov_forms_lts.schema.yml @@ -0,0 +1,12 @@ +# Schema for the configuration files of the localgov_forms_lts submodule. + +localgov_forms_lts.settings: + type: config_object + label: 'Webform submissions LTS config' + mapping: + is_copying_enabled: + type: boolean + label: 'Is copying to LTS database enabled?' + pii_redactor_plugin_id: + type: machine_name + label: 'PII redactor plugin id' diff --git a/modules/localgov_forms_lts/localgov_forms_lts.info.yml b/modules/localgov_forms_lts/localgov_forms_lts.info.yml new file mode 100644 index 0000000..efcf787 --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.info.yml @@ -0,0 +1,10 @@ +name: LocalGov Forms long term storage +type: module +description: Long term storage for Webform submissions. +core_version_requirement: ^10 || ^11 +php: 8.0 +package: LocalGov Drupal +lifecycle: experimental + +dependencies: +- webform:webform diff --git a/modules/localgov_forms_lts/localgov_forms_lts.install b/modules/localgov_forms_lts/localgov_forms_lts.install new file mode 100644 index 0000000..6994c61 --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.install @@ -0,0 +1,106 @@ + $table_schema) { + _localgov_forms_lts_copy_table($table_name, $table_schema, db: Constants::LTS_DB_KEY); + } +} + +/** + * Implements hook_requirements(). + * + * Checks for the presence of the localgov_forms_lts database. + */ +function localgov_forms_lts_requirements($phase) { + + $requirements = [ + 'localgov_forms_lts' => [ + 'title' => t('LocalGov Forms LTS'), + 'value' => t('Available'), + 'description' => t('LocalGov Forms LTS database available.'), + 'severity' => REQUIREMENT_OK, + ], + ]; + + if (!function_exists('localgov_forms_lts_has_db')) { + // Some necessary files have not been loaded yet during the "install" phase. + require_once __DIR__ . '/localgov_forms_lts.module'; + require_once __DIR__ . '/src/Constants.php'; + } + + if (!localgov_forms_lts_has_db()) { + $requirements['localgov_forms_lts']['value'] = t('Unavailable'); + $requirements['localgov_forms_lts']['description'] = t('The LocalGov Forms LTS database must exist for this module to function.'); + $requirements['localgov_forms_lts']['severity'] = REQUIREMENT_ERROR; + } + + return $requirements; +} + +/** + * Extracts entity storage schema. + * + * Returns the entity storage schema for the webform_submission content entity. + */ +function _localgov_forms_lts_get_webform_submission_storage_schema(): array { + + $entity_type_manager = Drupal::service('entity_type.manager'); + $entity_storage = $entity_type_manager->getStorage('webform_submission'); + $entity_type = $entity_storage->getEntityType(); + $entity_field_manager = Drupal::service('entity_field.manager'); + $db_service = Drupal::service('database'); + + // @phpstan-ignore-next-line Avoid incorrect type inference of $entity_type. + $entity_schema = (new class($entity_type_manager, $entity_type, $entity_storage, $db_service, $entity_field_manager) extends WebformSubmissionStorageSchema { + + /** + * Public wrapper over protected method. + */ + public function getEntitySchemaWrapper(ContentEntityTypeInterface $entity_type) { + + return parent::getEntitySchema($entity_type); + } + + })->getEntitySchemaWrapper($entity_type); + + return $entity_schema; +} + +/** + * Creates database tables. + */ +function _localgov_forms_lts_copy_table(string $table_name, array $table_schema, string $db): void { + + $db_connection = Database::getConnection(key: $db); + $tx = $db_connection->startTransaction(); + + try { + $db_connection->schema()->createTable($table_name, $table_schema); + } + catch (Exception $e) { + $tx->rollBack(); + } +} diff --git a/modules/localgov_forms_lts/localgov_forms_lts.links.task.yml b/modules/localgov_forms_lts/localgov_forms_lts.links.task.yml new file mode 100644 index 0000000..f3ad60b --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.links.task.yml @@ -0,0 +1,27 @@ +# Submissions > LTS +entity.webform_submission.lts_collection: + title: 'LTS' + route_name: entity.webform_submission.lts_collection + parent_id: entity.webform_submission.collection + weight: 21 + +# View +entity.webform_submission.lts_view: + title: 'View' + route_name: entity.webform_submission.lts_view + base_route: entity.webform_submission.lts_view + weight: 1 + +# Notes +entity.webform_submission.lts_notes: + title: 'Notes' + route_name: entity.webform_submission.lts_notes + base_route: entity.webform_submission.lts_view + weight: 3 + +# Config tab +localgov_forms_lts.lts_config: + title: 'LTS' + route_name: localgov_forms_lts.lts_config + parent_id: webform.config + weight: 43 diff --git a/modules/localgov_forms_lts/localgov_forms_lts.module b/modules/localgov_forms_lts/localgov_forms_lts.module new file mode 100644 index 0000000..18ed91c --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.module @@ -0,0 +1,90 @@ +get('config.factory')->get(Constants::LTS_CONFIG_ID); + + $is_copying_enabled = $lts_config->get(Constants::LTS_CONFIG_COPY_STATE); + if (!$is_copying_enabled) { + return; + } + + $pii_redactor_plugin_id = $lts_config->get(Constants::LTS_CONFIG_PII_REDACTOR_PLUGIN_ID); + $pii_redactor_plugin = ($pii_redactor_plugin_id && $service_container->has(Constants::PII_REDACTOR_PLUGIN_MANAGER)) ? $service_container->get(Constants::PII_REDACTOR_PLUGIN_MANAGER)->createInstance($pii_redactor_plugin_id) : NULL; + + $lts_copy_obj = LtsCopy::create(Drupal::getContainer(), $pii_redactor_plugin); + $copy_results = $lts_copy_obj->copy(); + + $feedback_msg = _localgov_forms_lts_prepare_feedback_msg($copy_results); + Drupal::service('logger.factory') + ->get(Constants::LTS_LOGGER_CHANNEL_ID) + ->info($feedback_msg); +} + +/** + * Prepares feedback message for copying. + * + * The feedback message is prepared from the outcome of Webform submission copy + * operations. + */ +function _localgov_forms_lts_prepare_feedback_msg(array $copy_results): MarkupInterface { + + $copy_successes = array_filter($copy_results); + $copy_failures = array_diff_key($copy_results, $copy_successes); + + $successfully_copied_sid_list = array_keys($copy_successes); + $unsuccessfully_copied_sid_list = array_keys($copy_failures); + + $successfully_copied_sid_list_msg = $successfully_copied_sid_list ? implode(', ', $successfully_copied_sid_list) : 'None'; + $unsuccessfully_copied_sid_list_msg = $unsuccessfully_copied_sid_list ? implode(', ', $unsuccessfully_copied_sid_list) : 'None'; + + $feedback_msg = t('Successfully copied Webform submission ids: %successes. :newline Failed copies: %failures.', [ + '%successes' => $successfully_copied_sid_list_msg, + '%failures' => $unsuccessfully_copied_sid_list_msg, + ':newline' => PHP_EOL, + ]); + + return $feedback_msg; +} diff --git a/modules/localgov_forms_lts/localgov_forms_lts.routing.yml b/modules/localgov_forms_lts/localgov_forms_lts.routing.yml new file mode 100644 index 0000000..29d41de --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.routing.yml @@ -0,0 +1,38 @@ +# List of Webform submissions from Long term storage. +entity.webform_submission.lts_collection: + path: '/admin/structure/webform/submissions/lts/{submission_view}' + defaults: + _controller: '\Drupal\localgov_forms_lts\WebformSubmissionLtsListBuilder::render' + _title: 'Webforms: Submissions' + submission_view: '' + requirements: + _custom_access: '\Drupal\webform\Access\WebformAccountAccess:checkSubmissionAccess' + +# Individual Webform submission view from Long term storage. +entity.webform_submission.lts_view: + path: '/admin/structure/webform/manage/{webform}/submission/{webform_sid}/lts' + defaults: + _controller: '\Drupal\localgov_forms_lts\Controller\WebformSubmissionLtsViewController::viewFromLts' + _title_callback: '\Drupal\localgov_forms_lts\Controller\WebformSubmissionLtsViewController::titleFromLts' + view_mode: 'html' + requirements: + _custom_access: '\Drupal\webform\Access\WebformAccountAccess:checkSubmissionAccess' + +# Individual Webform submission notes from Long term storage. +entity.webform_submission.lts_notes: + path: '/admin/structure/webform/manage/{webform}/submission/{webform_sid}/notes/lts' + defaults: + _controller: '\Drupal\localgov_forms_lts\Controller\WebformSubmissionLtsViewController::noteViewFromLts' + _title_callback: '\Drupal\localgov_forms_lts\Controller\WebformSubmissionLtsViewController::titleFromLts' + view_mode: 'html' + requirements: + _custom_access: '\Drupal\webform\Access\WebformAccountAccess:checkSubmissionAccess' + +# Config form. +localgov_forms_lts.lts_config: + path: '/admin/structure/webform/config/submissions-lts' + defaults: + _form: '\Drupal\localgov_forms_lts\Form\LTSSettingsForm' + _title: 'Webform submissions LTS' + requirements: + _permission: 'administer site configuration' diff --git a/modules/localgov_forms_lts/localgov_forms_lts.services.yml b/modules/localgov_forms_lts/localgov_forms_lts.services.yml new file mode 100644 index 0000000..6440676 --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.services.yml @@ -0,0 +1,12 @@ +services: + localgov_forms_lts_db: + class: Drupal\Core\Database\Connection + factory: Drupal\Core\Database\Database::getConnection + arguments: + $key: localgov_forms_lts + + localgov_forms_lts.query.sql: + class: Drupal\Core\Entity\Query\Sql\QueryFactory + arguments: ['@localgov_forms_lts_db'] + tags: + - { name: backend_overridable } diff --git a/modules/localgov_forms_lts/src/Constants.php b/modules/localgov_forms_lts/src/Constants.php new file mode 100644 index 0000000..dc41f3c --- /dev/null +++ b/modules/localgov_forms_lts/src/Constants.php @@ -0,0 +1,70 @@ +ltsStorage->load($webform_sid); + return parent::view($webform_sub, $view_mode, $langcode); + } + + /** + * Webform submission notes callback. + * + * Loads the Webform submission from Long term storage. + */ + public function noteViewFromLts(int $webform_sid, $view_mode = 'default', $langcode = NULL) { + + $webform_sub = $this->ltsStorage->load($webform_sid); + return [ + '#markup' => '
' . $webform_sub->getNotes() . '', + ]; + } + + /** + * Entity title callback. + * + * Loads the Webform submission from Long term storage. + */ + public function titleFromLts(int $webform_sid, $duplicate = FALSE) { + + $webform_sub = $this->ltsStorage->load($webform_sid); + return parent::title($webform_sub, $duplicate); + } + + /** + * Factory. + */ + public static function create(ContainerInterface $container) { + + $instance = parent::create($container); + + $webform_sub_entity_type = $container->get('entity_type.manager')->getDefinition('webform_submission'); + $instance->ltsStorage = LtsStorageForWebformSubmission::createInstance($container, $webform_sub_entity_type); + + return $instance; + } + + /** + * Database service for the Long term storage database. + * + * @var Drupal\webform\WebformSubmissionStorageInterface + */ + protected $ltsStorage; + +} diff --git a/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php b/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php new file mode 100644 index 0000000..667609f --- /dev/null +++ b/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php @@ -0,0 +1,97 @@ + FALSE]) { + + if (!localgov_forms_lts_has_db()) { + $this->logger->error(dt('The LocalGov Forms LTS database must exist for this Drush command to function.')); + return; + } + + $is_proceed = $options['force'] ?: $this->configFactory->get(Constants::LTS_CONFIG_ID)?->get(Constants::LTS_CONFIG_COPY_STATE); + if (!$is_proceed) { + $this->logger->warning(dt('Copying is disabled in localgov_forms_lts module configuration. Use --force to override.')); + return; + } + + $pii_redactor_plugin_id = $this->configFactory->get(Constants::LTS_CONFIG_ID)?->get(Constants::LTS_CONFIG_PII_REDACTOR_PLUGIN_ID); + $pii_redactor_plugin = ($pii_redactor_plugin_id && $this->serviceContainer->has(Constants::PII_REDACTOR_PLUGIN_MANAGER)) ? $this->serviceContainer->get(Constants::PII_REDACTOR_PLUGIN_MANAGER)->createInstance($pii_redactor_plugin_id) : NULL; + + $lts_copy_obj = LtsCopy::create(\Drupal::getContainer(), $pii_redactor_plugin); + $webform_sub_ids_to_copy = $lts_copy_obj->findCopyTargets(); + $batch_count = ceil(count($webform_sub_ids_to_copy) / Constants::COPY_LIMIT); + + $batch_builder = new BatchBuilder(); + $drupal_logger = $this->drupalLoggerFactory->get(Constants::LTS_LOGGER_CHANNEL_ID); + for ($i = 0; $i < $batch_count; $i++) { + $batch_builder->addOperation([self::class, 'copyInBatch'], [ + $pii_redactor_plugin, + $drupal_logger, + ]); + } + + batch_set($batch_builder->toArray()); + drush_backend_batch_process(); + + // Info messages are not appearing in the console, so settling for Notices. + $this->logger->notice('Batch operation for copying Webform submissions to Long term storage ends.'); + } + + /** + * Batch operation callback. + * + * Copies a fixed number of Webform submissions to LTS. + */ + public static function copyInBatch(?PluginInspectionInterface $pii_redactor_plugin, LoggerInterface $drupal_logger, &$context) { + + $lts_copy_obj = LtsCopy::create(\Drupal::getContainer(), $pii_redactor_plugin); + $copy_results = $lts_copy_obj->copy(); + + $feedback = _localgov_forms_lts_prepare_feedback_msg($copy_results); + $drupal_logger->info($feedback); + + $context['results'][] = $copy_results; + $context['message'] = $feedback; + } + + /** + * Constructs a LocalgovFormsLtsCommands object. + */ + public function __construct( + private readonly ConfigFactoryInterface $configFactory, + private readonly LoggerChannelFactoryInterface $drupalLoggerFactory, + private readonly ContainerInterface $serviceContainer, + ) { + parent::__construct(); + } + +} diff --git a/modules/localgov_forms_lts/src/Form/LTSSettingsForm.php b/modules/localgov_forms_lts/src/Form/LTSSettingsForm.php new file mode 100644 index 0000000..09bbfd4 --- /dev/null +++ b/modules/localgov_forms_lts/src/Form/LTSSettingsForm.php @@ -0,0 +1,74 @@ + 'radios', + '#title' => $this->t('Activate?'), + '#description' => $this->t('Activates copying Webform submissions to the Long Term Storage (LTS) database.'), + '#config_target' => Constants::LTS_CONFIG_ID . ':' . Constants::LTS_CONFIG_COPY_STATE, + '#options' => [ + TRUE => $this->t('Yes'), + FALSE => $this->t('No'), + ], + ]; + + $pii_redactor_plugin_id_list = $this->optionalPIIRedactorPluginManager ? array_map(fn(array $def): string => $def['label'], $this->optionalPIIRedactorPluginManager->getDefinitions()) : []; + $form[Constants::LTS_CONFIG_PII_REDACTOR_PLUGIN_ID] = [ + '#type' => 'select', + '#title' => $this->t('PII redactor plugin'), + '#description' => $this->t('Select a plugin to redact Personally Identifiable Information (PII) while copying to LTS database.'), + '#config_target' => Constants::LTS_CONFIG_ID . ':' . Constants::LTS_CONFIG_PII_REDACTOR_PLUGIN_ID, + '#options' => $pii_redactor_plugin_id_list, + '#empty_value' => '', + ]; + + return parent::buildForm($form, $form_state); + } + + /** + * {@inheritdoc} + */ + public function getFormId() { + + return 'localgov_forms_lts_settings'; + } + + /** + * Keeps track of the optional PII redactor plugin manager. + */ + public function __construct(protected ?PluginManagerInterface $optionalPIIRedactorPluginManager) {} + + /** + * Factory. + * + * If the PII redactor plugin manager is available, passes it to the + * constructor. + */ + public static function create(ContainerInterface $container) { + + $pii_redactor_plugin_manager = $container->has(Constants::PII_REDACTOR_PLUGIN_MANAGER) ? $container->get(Constants::PII_REDACTOR_PLUGIN_MANAGER) : NULL; + + return new static($pii_redactor_plugin_manager); + } + +} diff --git a/modules/localgov_forms_lts/src/LtsCopy.php b/modules/localgov_forms_lts/src/LtsCopy.php new file mode 100644 index 0000000..afdfa2f --- /dev/null +++ b/modules/localgov_forms_lts/src/LtsCopy.php @@ -0,0 +1,209 @@ +findLastCopiedSubId(); + $webform_subs_to_copy = $this->findCopyTargets($count); + + $copy_results = []; + $has_copied = FALSE; + foreach ($webform_subs_to_copy as $webform_sub_id) { + $is_new_webform_sub = $webform_sub_id > $last_copied_webform_sub_id; + $copy_results[$webform_sub_id] = $this->copySub((int) $webform_sub_id, $is_new_webform_sub); + $has_copied = TRUE; + } + + if ($has_copied) { + $this->setLatestUpdateTimestamp($copy_results); + } + + return $copy_results; + } + + /** + * Saves a single webform submission in LTS database. + */ + public function copySub(int $webform_sub_id, bool $is_new_webform_sub) :bool { + + $webform_sub = $this->webformSubStorage->load($webform_sub_id); + + if ($this->optionalPIIRedactionPlugin) { + $this->optionalPIIRedactionPlugin->redact($webform_sub); + } + + $db_connection = $this->ltsStorage->getDatabaseConnection(); + $tx = $db_connection->startTransaction(); + + try { + if ($is_new_webform_sub) { + $this->ltsStorage->resave($webform_sub->enforceIsNew()); + } + else { + $this->ltsStorage->resave($webform_sub); + } + } + catch (\Exception $e) { + $tx->rollBack(); + + $this->ltsLogger->error('Failed to add/edit Webform submission: %sub-id', [ + '%sub-id' => $webform_sub_id, + ]); + + return FALSE; + } + + return TRUE; + } + + /** + * Finds last copied Webform submission's id. + */ + public function findLastCopiedSubId() :int { + + $last_copied_webform_sub_id_raw = $this->ltsStorage->getAggregateQuery() + ->accessCheck(FALSE) + ->aggregate('sid', 'MAX') + ->execute(); + $last_copied_webform_sub_id = $last_copied_webform_sub_id_raw[0]['sid_max'] ?? 0; + return (int) $last_copied_webform_sub_id; + } + + /** + * Finds Webform submissions to copy. + * + * These are the Webform submissions that have been added or edited since the + * last copy operation. + * + * For offset to work, *both* parameters must be provided with nonnegative + * values. + */ + public function findCopyTargets(int $count = -1) :array { + + $last_copied_webform_sub_changed_ts = $this->findLatestUpdateTimestamp(); + + $webform_subs_to_copy_query = $this->webformSubStorage + ->getQuery() + ->accessCheck(FALSE) + ->condition('changed', $last_copied_webform_sub_changed_ts, '>') + ->condition('in_draft', 0) + ->sort('changed'); + + if ($count > -1) { + $webform_subs_to_copy_query->range(start: 0, length: $count); + } + + $copy_targets = $webform_subs_to_copy_query->execute(); + return $copy_targets; + } + + /** + * When did the last copied Webform submission change? + */ + public function findLatestUpdateTimestamp() :int { + + $ts = (int) $this->ltsKeyValueStore->get(Constants::LAST_CHANGE_TIMESTAMP, default: 0); + return $ts; + } + + /** + * Records time of latest copy operation. + */ + public function setLatestUpdateTimestamp(array $copy_results) :void { + + $last_copied_webform_sub_id = array_key_last($copy_results); + $last_copied_webform_sub = $this->webformSubStorage->load($last_copied_webform_sub_id); + $this->ltsKeyValueStore->set(Constants::LAST_CHANGE_TIMESTAMP, $last_copied_webform_sub->getChangedTime()); + } + + /** + * Constructor. + * + * Keeps track of dependencies. + */ + public function __construct(EntityTypeManagerInterface $entity_type_manager, KeyValueFactoryInterface $key_value_factory, LoggerChannelFactoryInterface $logger_factory, WebformSubmissionStorageInterface $lts_storage, ?PluginInspectionInterface $pii_redaction_plugin = NULL) { + + $this->webformSubStorage = $entity_type_manager->getStorage('webform_submission'); + $this->ltsStorage = $lts_storage; + $this->ltsKeyValueStore = $key_value_factory->get(Constants::LTS_KEYVALUE_STORE_ID); + $this->ltsLogger = $logger_factory->get(Constants::LTS_LOGGER_CHANNEL_ID); + $this->optionalPIIRedactionPlugin = $pii_redaction_plugin; + } + + /** + * Factory. + */ + public static function create(ContainerInterface $container, ?PluginInspectionInterface $pii_redaction_plugin = NULL) :LtsCopy { + + $webform_sub_def = $container->get('entity_type.manager')->getDefinition('webform_submission'); + + return new LtsCopy( + $container->get('entity_type.manager'), + $container->get('keyvalue'), + $container->get('logger.factory'), + LtsStorageForWebformSubmission::createInstance($container, $webform_sub_def), + $pii_redaction_plugin, + ); + } + + /** + * Key value store for LTS related state. + * + * @var Drupal\Core\KeyValueStore\KeyValueStoreInterface + */ + protected $ltsKeyValueStore; + + /** + * Entity type manager service. + * + * @var Drupal\webform\WebformSubmissionStorageInterface + */ + protected $webformSubStorage; + + /** + * Database service for the Long term storage database. + * + * @var Drupal\webform\WebformSubmissionStorageInterface + */ + protected $ltsStorage; + + /** + * Logger channel. + * + * @var Drupal\Core\Logger\LoggerChannelInterface + */ + protected $ltsLogger; + + /** + * Optional PII redactor plugin manager. + * + * @var Drupal\Component\Plugin\PluginInspectionInterface + */ + protected $optionalPIIRedactionPlugin = NULL; + +} diff --git a/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php b/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php new file mode 100644 index 0000000..241f009 --- /dev/null +++ b/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php @@ -0,0 +1,89 @@ +setLtsDatabaseConnection($lts_db_connection); // Optional. + * $a_webform_submission = $lts_storage->load($a_webform_submission_id); + * @endcode + */ +class LtsStorageForWebformSubmission extends WebformSubmissionStorage { + + /** + * Constructor wrapper. + * + * - Switches to the LTS database. + */ + public function __construct(...$args) { + + parent::__construct(...$args); + + $this->database = Database::getConnection(key: Constants::LTS_DB_KEY); + } + + /** + * Setter for database connection. + */ + public function setDatabaseConnection(DbConnection $db_connection): void { + + $this->database = $db_connection; + } + + /** + * Getter for database connection. + */ + public function getDatabaseConnection(): DbConnection { + + return $this->database; + } + + /** + * Disables persistent cache. + * + * Because we do not have any in LTS. + */ + protected function getFromPersistentCache(?array &$ids = NULL) { + + return []; + } + + /** + * See above. + */ + protected function setPersistentCache($entities) {} + + /** + * Customizes cache Ids for LTS. + * + * Although we have disabled persistent cache above, cache ids are still used + * in static cache. + */ + protected function buildCacheId($id) { + + return Constants::LTS_CACHE_ID_PREFIX . ":{$this->entityTypeId}:$id"; + } + + /** + * {@inheritdoc} + * + * Names our custom entity query service that speaks to the LTS database. + */ + protected function getQueryServiceName() { + + return Constants::LTS_ENTITY_QUERY_SERVICE; + } + +} diff --git a/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php b/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php new file mode 100644 index 0000000..4192694 --- /dev/null +++ b/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php @@ -0,0 +1,82 @@ +id(); + $webform_id = $webform_submission->getWebform()->id(); + + $ops = parent::getDefaultOperations($entity); + + $lts_ops = [ + 'view' => $ops['view'] ?? [], + 'notes' => $ops['notes'] ?? [], + ]; + $lts_ops['view']['url'] = Url::fromRoute('entity.webform_submission.lts_view', [ + 'webform' => $webform_id, + 'webform_sid' => $webform_submission_id, + ]); + $lts_ops['notes']['url'] = Url::fromRoute('entity.webform_submission.lts_notes', [ + 'webform' => $webform_id, + 'webform_sid' => $webform_submission_id, + ]); + + return $lts_ops; + } + + /** + * {@inheritdoc} + * + * Tells the list builder to use our Webform submissions LTS storage. + */ + public static function createInstance(ContainerInterface $container, EntityTypeInterface $entity_type) { + /** @var \Drupal\webform\WebformSubmissionLtsListBuilder $instance */ + $instance = parent::createInstance($container, $entity_type); + + $lts_storage = LtsStorageForWebformSubmission::createInstance($container, $entity_type); + $instance->storage = $lts_storage; + $instance->initialize(); + $instance->columns = $instance->storage->getSubmissionsColumns(); + + return $instance; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + + $webform_sub_def = $container->get('entity_type.manager')->getDefinition('webform_submission'); + return self::createInstance($container, $webform_sub_def); + } + +} diff --git a/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php b/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php new file mode 100644 index 0000000..ee106a3 --- /dev/null +++ b/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php @@ -0,0 +1,124 @@ +container->get('entity_type.manager')->getStorage('webform')->load(self::TEST_WEBFORM_ID); + $this->assertNotNull($contact2_webform); + + $a_webform_submission = WebformSubmission::create([ + 'webform_id' => self::TEST_WEBFORM_ID, + 'data' => [ + 'name' => 'Foo Bar', + 'email' => 'foo@example.net', + ], + ]); + + // Temporary measure to satisfy + // LtsStorageForWebformSubmission::__construct(). + // Gets overwritten by the call to $test_obj->setDatabaseConnection() below. + Database::addConnectionInfo(Constants::LTS_DB_KEY, 'default', [ + 'driver' => 'fake_lts', + 'namespace' => 'Drupal\\localgov_forms_test\\Driver\\Database\\FakeLts', + ]); + $webform_sub_def = $this->container->get('entity_type.manager')->getDefinition('webform_submission'); + $test_obj = LtsStorageForWebformSubmission::createInstance($this->container, $webform_sub_def); + $test_obj->setDatabaseConnection($this->mockLtsDbConnection); + + $test_obj->resave($a_webform_submission); + } + + /** + * Prepares the mock LTS database connection. + */ + protected function setUp(): void { + + parent::setUp(); + + $this->installSchema('webform', ['webform']); + $this->installConfig(['webform']); + $this->installEntitySchema('user'); + + $mock_insert_query = $this->createConfiguredMock(Insert::class, [ + 'execute' => random_int(1, 10), + ]); + $mock_insert_query->method('fields')->willReturnSelf(); + + $this->mockLtsDbConnection = $this->createMock(DbConnection::class); + $this->mockLtsDbConnection->expects($this->exactly(self::WEBFORM_SUB_LTS_INSERT_COUNT)) + ->method('insert') + ->willReturnMap([ + ['webform_submission', $mock_insert_query], + ['webform_submission', [ + 'return' => self::DEPRECATED_D10_RETURN_INSERT_ID, + ], $mock_insert_query, + ], + ['webform_submission_data', $mock_insert_query], + ['webform_submission_data', [], $mock_insert_query], + ]); + } + + /** + * A new Webform submission is inserted into the LTS database twice. + * + * Once in the webform_submission table and then in the + * webform_submission_data table. + */ + const WEBFORM_SUB_LTS_INSERT_COUNT = 2; + + /** + * A Webform from the localgov_forms_test module. + */ + const TEST_WEBFORM_ID = 'contact'; + + /** + * Mirrors the deprecated Database::RETURN_INSERT_ID. + */ + const DEPRECATED_D10_RETURN_INSERT_ID = 3; + + /** + * Modules to enable. + * + * @var array + */ + protected static $modules = [ + 'localgov_forms_test', + 'system', + 'user', + 'webform', + ]; + + /** + * Mock LTS database connection. + * + * Is it being used at all? That's what this test is about. + * + * @var Drupal\Core\Database\Connection + */ + protected $mockLtsDbConnection; + +} diff --git a/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php b/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php new file mode 100644 index 0000000..9ea869d --- /dev/null +++ b/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php @@ -0,0 +1,161 @@ +mockEntityTypeManager, $this->mockLtsKeyValueFactory, $this->mockLtsLoggerFactory, $this->mockLtsStorage); + + $copy_results = $test_obj->copy(); + + $this->assertCount(self::WEBFORM_SUB_EXPECTED_COPY_COUNT, $copy_results); + } + + /** + * Creates mock dependencies. + * + * Initializes all objects needed to create an LtsCopy object. + */ + public function setUp(): void { + + parent::setUp(); + + $mock_webform_sub_storage = static::setupMockWebformSubmissionStorage(); + $this->mockEntityTypeManager = $this->createConfiguredMock(EntityTypeManagerInterface::class, [ + 'getStorage' => $mock_webform_sub_storage, + ]); + + $mock_lts_storage_query = $this->createMock(QueryAggregateInterface::class); + $mock_lts_storage_query->method('execute') + ->willReturn([['sid_max' => self::LAST_COPIED_WEBFORM_SUB_ID]]); + $mock_lts_storage_query->method('accessCheck')->willReturnSelf(); + $mock_lts_storage_query->method('aggregate')->willReturnSelf(); + $this->mockLtsStorage = $this->createConfiguredMock(LtsStorageForWebformSubmission::class, [ + 'getAggregateQuery' => $mock_lts_storage_query, + 'getDatabaseConnection' => $this->createMock(DbConnection::class), + ]); + $this->mockLtsStorage->expects($this->exactly(self::WEBFORM_SUB_EXPECTED_COPY_COUNT))->method('resave'); + + $this->mockLtsKeyValueFactory = $this->createConfiguredMock(KeyValueFactoryInterface::class, [ + 'get' => $this->createMock(KeyValueStoreInterface::class), + ]); + + $this->mockLtsLoggerFactory = $this->createMock(LoggerChannelFactoryInterface::class); + } + + /** + * Prepares a mock Webform submission storage object. + * + * When queried, returns three mock Webform submission entities. + */ + public function setupMockWebformSubmissionStorage(): WebformSubmissionStorageInterface { + $mock_webform_sub_query = $this->createMock(QueryInterface::class); + $mock_webform_sub_query->expects($this->any()) + ->method('execute') + ->willReturn([ + self::LAST_COPIED_WEBFORM_SUB_ID => (string) self::LAST_COPIED_WEBFORM_SUB_ID, + self::NEW_WEBFORM_SUB_ID0 => (string) self::NEW_WEBFORM_SUB_ID0 , + self::NEW_WEBFORM_SUB_ID1 => (string) self::NEW_WEBFORM_SUB_ID1 , + ]); + $mock_webform_sub_query->method('accessCheck')->willReturnSelf(); + $mock_webform_sub_query->method('condition')->willReturnSelf(); + $mock_webform_sub_query->method('sort')->willReturnSelf(); + $mock_webform_sub_storage = $this->createConfiguredMock(WebformSubmissionStorageInterface::class, [ + 'getQuery' => $mock_webform_sub_query, + ]); + + $mock_webform = $this->createConfiguredMock(WebformInterface::class, [ + 'getElementsDecodedAndFlattened' => [], + ]); + $mock_existing_webform_sub = $this->createConfiguredMock(WebformSubmissionInterface::class, [ + 'id' => self::LAST_COPIED_WEBFORM_SUB_ID, + 'getWebform' => $mock_webform, + ]); + $mock_new_webform_sub0 = $this->createConfiguredMock(WebformSubmissionInterface::class, [ + 'id' => self::NEW_WEBFORM_SUB_ID0, + 'getWebform' => $mock_webform, + ]); + $mock_new_webform_sub0->expects($this->once())->method('enforceIsNew')->willReturnSelf(); + $mock_new_webform_sub1 = $this->createConfiguredMock(WebformSubmissionInterface::class, [ + 'id' => self::NEW_WEBFORM_SUB_ID1, + 'getWebform' => $mock_webform, + ]); + $mock_new_webform_sub1->expects($this->once())->method('enforceIsNew')->willReturnSelf(); + $mock_webform_sub_storage->expects($this->exactly(self::WEBFORM_SUB_EXPECTED_LOAD_COUNT)) + ->method('load') + ->willReturnMap([ + [self::LAST_COPIED_WEBFORM_SUB_ID, $mock_existing_webform_sub], + [self::NEW_WEBFORM_SUB_ID0, $mock_new_webform_sub0], + [self::NEW_WEBFORM_SUB_ID1, $mock_new_webform_sub1], + ]); + + return $mock_webform_sub_storage; + } + + const LAST_COPIED_WEBFORM_SUB_ID = 99; + + const NEW_WEBFORM_SUB_ID0 = 100; + + const NEW_WEBFORM_SUB_ID1 = 101; + + const WEBFORM_SUB_EXPECTED_COPY_COUNT = 3; + + const WEBFORM_SUB_EXPECTED_LOAD_COUNT = 4; + + /** + * Mock KeyValue factory. + * + * @var \Drupal\Core\KeyValueStore\KeyValueFactoryInterface + */ + protected $mockLtsKeyValueFactory; + + /** + * Mock EntityTypeManager. + * + * @var \Drupal\Core\Entity\EntityTypeManagerInterface + */ + protected $mockEntityTypeManager; + + /** + * Mock Long term webform submission storage. + * + * @var \Drupal\webform\WebformSubmissionStorageInterface + */ + protected $mockLtsStorage; + + /** + * Mock logger. + * + * @var \Drupal\Core\Logger\LoggerChannelFactoryInterface + */ + protected $mockLtsLoggerFactory; + +} diff --git a/src/Annotations/PIIRedactor.php b/src/Annotations/PIIRedactor.php new file mode 100644 index 0000000..8216c29 --- /dev/null +++ b/src/Annotations/PIIRedactor.php @@ -0,0 +1,38 @@ +getElementData($elem)) { + $webform_sub->setElementData($elem, NULL); + + return $elem; + } + }, $elems_to_redact['full']); + + $partial_redaction_result = array_map(function ($elem) use ($webform_sub) { + if ($text = $webform_sub->getElementData($elem)) { + [$redacted_text, $redaction_count] = BestEffortPIIRedactorForText::redact($text); + + if ($redaction_count) { + $webform_sub->setElementData($elem, $redacted_text); + return $elem; + } + } + }, $elems_to_redact['part']); + + $redacted_elems = array_filter($redaction_result); + $partly_redacted_elems = array_filter($partial_redaction_result); + static::addRedactionNote($webform_sub, $redacted_elems); + static::addRedactionNote($webform_sub, $partly_redacted_elems, note_prefix: 'Partly redacted elements: '); + + $all_redacted_elems = [...$redacted_elems, ...$partly_redacted_elems]; + return $all_redacted_elems; + } + + /** + * Finds the Webform element names to redact. + * + * The result array contains two keys: + * - full: These elements are to be fully redacted. + * - part: The values of these elements contain PII among other text and are + * to be partly redacted. + */ + public static function findElemsToRedact(WebformSubmissionInterface $webform_sub) :array { + + $elem_type_mapping = static::listElemsAndTypes($webform_sub); + $pii_mapping = array_intersect($elem_type_mapping, static::PII_ELEMENT_TYPES); + $pii_elems = array_keys($pii_mapping); + + $potential_mapping = array_intersect($elem_type_mapping, static::POTENTIAL_PII_ELEMENT_TYPES); + $guessed_pii_elems = preg_grep(static::GUESSED_PII_ELEM_PATTERN, array_keys($potential_mapping)); + + $elems_w_some_pii = array_keys(array_intersect($elem_type_mapping, static::PII_ELEMENT_TYPES_TO_REDUCT_IN_PART)); + + $elems_to_redact = [ + 'full' => array_unique([...$pii_elems, ...$guessed_pii_elems]), + 'part' => $elems_w_some_pii, + ]; + return $elems_to_redact; + } + + /** + * Prepares mapping of element ids and types. + */ + public static function listElemsAndTypes(WebformSubmissionInterface $webform_sub) :array { + + $elems = $webform_sub->getWebform()->getElementsDecodedAndFlattened(); + return array_map(fn($elem_def) => $elem_def['#type'], $elems); + } + + /** + * Adds redaction note. + * + * Adds a note to the Webform submission to highlight the redacted elements. + */ + public static function addRedactionNote(WebformSubmissionInterface $webform_sub, array $redacted_elems, string $note_prefix = 'Redacted elements: '): void { + + if (empty($redacted_elems)) { + return; + } + + $redaction_note = $note_prefix . implode(', ', $redacted_elems) . '.'; + $existing_note = $webform_sub->getNotes(); + + $note_list = array_filter([$existing_note, $redaction_note]); + $webform_sub->setNotes(implode(PHP_EOL, $note_list)); + } + + /** + * Element types carrying PII for certain. + */ + const PII_ELEMENT_TYPES = [ + 'address', + 'email', + 'localgov_forms_dob', + 'localgov_webform_uk_address', + 'number', + 'tel', + 'webform_name', + 'webform_address', + 'webform_contact', + 'webform_telephone', + ]; + + /** + * Element types that *may* carry PII. + */ + const POTENTIAL_PII_ELEMENT_TYPES = [ + 'localgov_forms_date', + 'checkboxes', + 'processed_text', + 'radios', + 'textfield', + ]; + + /** + * Element types with PII mixed with other text. + */ + const PII_ELEMENT_TYPES_TO_REDUCT_IN_PART = [ + 'textarea', + ]; + + /** + * Preg pattern. + * + * Element type naming pattern indicating possible link with PII. + */ + const GUESSED_PII_ELEM_PATTERN = '#name|mail|phone|contact_number|date_of_birth|dob_|nino|address|postcode|post_code|personal_|title|gender|sex|ethnicity|passport|serial_number|reg_number|pcn_|driver_#i'; + +} diff --git a/src/BestEffortPIIRedactorForText.php b/src/BestEffortPIIRedactorForText.php new file mode 100644 index 0000000..882def7 --- /dev/null +++ b/src/BestEffortPIIRedactorForText.php @@ -0,0 +1,92 @@ + 'textfield', - '#title' => t('Postcode or Street'), + '#title' => t('Postcode or street'), '#description' => $element['#address_search_description'] ?? t('Enter the postcode…'), + '#required_error' => t('You must enter a postcode or street.'), '#maxlength' => 64, '#size' => 64, '#weight' => '0', @@ -148,6 +165,7 @@ public static function processAddressLookupElement(&$element, FormStateInterface $element['address_select']['address_select_list'] = [ '#type' => 'select', '#title' => $element['#address_select_title'] ?? t('Select the address'), + '#required_error' => t('You must select an address.'), '#options' => [], '#empty_option' => '-' . t('Please choose an address') . '-', '#empty_value' => 0, @@ -173,7 +191,7 @@ public static function processAddressLookupElement(&$element, FormStateInterface $parent_container = $parent_container[$keyval]; } - // Extract the parent values form container. + // Extract the parent values from container. $parent_container_values = $form_values; foreach ($parents as $keyval) { $parent_container_values = $parent_container_values[$keyval]; @@ -301,14 +319,14 @@ public static function addressSelectLookup(string $address_search, array $addres // Get the address type to lookup. $address_type = $address_element['address_select']['address_select_list']['#address_type']; + $local_custodian_code = $address_element['#local_custodian_code']; + $selected_plugin_ids = $address_element['#geocoder_plugins']; // Do address lookup. // If its searching for the same address, return the static version. // Else make a new request. // This is to avoid multiple api lookup calls. - if ($address_search !== self::$searchString || $address_type !== self::$addressType) { - $selected_plugin_ids = $address_element['#geocoder_plugins']; - $local_custodian_code = $address_element['#local_custodian_code']; + if ($address_search !== self::$searchString || $address_type !== self::$addressType || $local_custodian_code !== self::$localCustodianCode || $selected_plugin_ids !== self::$selectedGeocoderPluginIds) { self::$addressResults = \Drupal::service('localgov_forms.address_lookup')->search([$address_search], $selected_plugin_ids, $local_custodian_code); } $address_list = self::$addressResults; @@ -348,8 +366,10 @@ public static function addressSelectLookup(string $address_search, array $addres unset($address_element['address_select']['address_select_list']['#suffix']); unset($address_element['address_select']['error']); - self::$searchString = $address_search; - self::$addressType = $address_type; + self::$searchString = $address_search; + self::$addressType = $address_type; + self::$localCustodianCode = $local_custodian_code; + self::$selectedGeocoderPluginIds = $selected_plugin_ids; return $address_element['address_select']['address_select_list']; } diff --git a/src/Element/UKAddressLookup.php b/src/Element/UKAddressLookup.php index 0549785..346d156 100644 --- a/src/Element/UKAddressLookup.php +++ b/src/Element/UKAddressLookup.php @@ -59,6 +59,9 @@ public static function getCompositeElements(array $element) { $element_list['address_1']['#prefix'] = '