diff --git a/documentation/revision-history-develop.md b/documentation/revision-history-develop.md index e965c6c2fa..bed3940c06 100644 --- a/documentation/revision-history-develop.md +++ b/documentation/revision-history-develop.md @@ -300,4 +300,7 @@ bugfix release: # 8.9.2 - 17.10.2025 DEVELOP - add ownerLifeCycleState -- add manageable ownerLifeCycleState menu \ No newline at end of file +- add manageable ownerLifeCycleState menu + +# 8.9.3 - 05.11.2025 DEVELOP +- hotfix missing permissions for app data import in certain constellations diff --git a/documentation/revision-history-main.md b/documentation/revision-history-main.md index 47790f5e8f..6c7e43c48a 100644 --- a/documentation/revision-history-main.md +++ b/documentation/revision-history-main.md @@ -552,3 +552,13 @@ hotfix release - fixing services-other ip proto import - improved quality control with stricter automated checks - various fixes in modelling module + +# 8.9.1 - 02.10.2025 MAIN +- owner-recertification + +# 8.9.2 - 17.10.2025 MAIN +- add ownerLifeCycleState +- add manageable ownerLifeCycleState menu + +# 8.9.3 - 05.11.2025 MAIN +- hotfix missing permissions for app data import in certain constellations diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 256db00cb6..41f19fad16 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -1,5 +1,5 @@ ### general settings -product_version: "8.9.2" +product_version: "8.9.3" ansible_user: "{{ lookup('env', 'USER') }}" ansible_become_method: sudo ansible_python_interpreter: /usr/bin/python3 @@ -59,6 +59,7 @@ debian_testing_version: "12" # should actually be 13 but microsoft does not yet provide https://packages.microsoft.com/config/debian/13 debian_testing_release_name: trixie arch: x86_64 +linux_architecture: amd64 redhat_major_version: "8" redhat_arch: "{{ redhat_major_version }}-{{ arch }}" diff --git a/inventory/group_vars/apiserver.yml b/inventory/group_vars/apiserver.yml index 06e15f1601..afff9277d3 100644 --- a/inventory/group_vars/apiserver.yml +++ b/inventory/group_vars/apiserver.yml @@ -8,7 +8,7 @@ api_hasura_admin_test_password: "not4production" api_user_email: "{{ api_user }}@{{ api_network_listening_ip_address }}" api_home: "{{ fworch_home }}/api" api_hasura_cli_bin: "{{ fworch_home }}/api/bin/hasura" -api_hasura_version: "v2.48.5" +api_hasura_version: "v2.48.6" api_project_name: api api_no_metadata: false api_rollback_is_running: false diff --git a/roles/api/files/replace_metadata.json b/roles/api/files/replace_metadata.json index abe5f938b5..079b8edde3 100644 --- a/roles/api/files/replace_metadata.json +++ b/roles/api/files/replace_metadata.json @@ -10033,20 +10033,25 @@ "check": {}, "columns": [ "active", - "app_id_external", "common_service_possible", + "is_default", + "recert_active", + "app_id_external", "criticality", "dn", "group_dn", - "id", "import_source", - "is_default", - "last_recert_check", + "last_recertifier_dn", "name", - "recert_active", "recert_check_params", + "id", + "last_recertifier", + "owner_lifecycle_state_id", "recert_interval", - "tenant_id" + "tenant_id", + "last_recert_check", + "last_recertified", + "next_recert_date" ] }, "comment": "" @@ -10443,20 +10448,25 @@ "permission": { "columns": [ "active", - "app_id_external", "common_service_possible", + "is_default", + "recert_active", + "app_id_external", "criticality", "dn", "group_dn", - "id", "import_source", - "is_default", - "last_recert_check", + "last_recertifier_dn", "name", - "recert_active", "recert_check_params", + "id", + "last_recertifier", + "owner_lifecycle_state_id", "recert_interval", - "tenant_id" + "tenant_id", + "last_recert_check", + "last_recertified", + "next_recert_date" ], "filter": {}, "check": null @@ -26195,4 +26205,4 @@ ] } } -} \ No newline at end of file +} diff --git a/roles/api/tasks/hasura-install.yml b/roles/api/tasks/hasura-install.yml index b32a975a67..7238bf48a4 100644 --- a/roles/api/tasks/hasura-install.yml +++ b/roles/api/tasks/hasura-install.yml @@ -2,84 +2,111 @@ - name: Install packages for python pip3 n virtualenv package: - name: "{{ item }}" - state: present + name: "{{ item }}" + state: present loop: - - python3-pip - - python3-virtualenv - - python3-docker + - python3-pip + - python3-virtualenv + - python3-docker become: true - name: read dbadmin pwd from secrets file slurp: - src: "{{ dbadmin_password_file }}" + src: "{{ dbadmin_password_file }}" register: api_user_password become: true - name: decode dbadmin pwd set_fact: - api_user_password: "{{ api_user_password['content'] | b64decode | trim }}" + api_user_password: "{{ api_user_password['content'] | b64decode | trim }}" - name: read jwt public key from file as JWT secret slurp: - src: "{{ jwt_public_key_file }}" + src: "{{ jwt_public_key_file }}" register: api_hasura_jwt_secret_dict become: true - name: decode key set_fact: - api_hasura_jwt_secret: "{{ api_hasura_jwt_secret_dict['content'] | b64decode }}" + api_hasura_jwt_secret: "{{ api_hasura_jwt_secret_dict['content'] | b64decode }}" - name: make sure {{ fworch_secrets_dir }} exists file: - path: "{{ fworch_secrets_dir }}" - state: directory - mode: "0750" - owner: "{{ fworch_user }}" - group: "{{ postgres_group }}" + path: "{{ fworch_secrets_dir }}" + state: directory + mode: "0750" + owner: "{{ fworch_user }}" + group: "{{ postgres_group }}" become: true - name: set static hasura admin pwd for test purposes only set_fact: - api_hasura_admin_secret: "{{ api_hasura_admin_test_password }}" + api_hasura_admin_secret: "{{ api_hasura_admin_test_password }}" when: testkeys is defined and testkeys|bool - name: set random hasura admin password set_fact: - api_hasura_admin_secret: "{{ randomly_generated_pwd }}" + api_hasura_admin_secret: "{{ randomly_generated_pwd }}" when: testkeys is not defined or not testkeys|bool - name: write hasura admin password to secrets directory copy: - content: "{{ api_hasura_admin_secret }}\n" - dest: "{{ fworch_secrets_dir }}/hasura_admin_pwd" - mode: '0600' - owner: "{{ fworch_user }}" - group: "{{ fworch_group }}" + content: "{{ api_hasura_admin_secret }}\n" + dest: "{{ fworch_secrets_dir }}/hasura_admin_pwd" + mode: "0600" + owner: "{{ fworch_user }}" + group: "{{ fworch_group }}" become: true - name: check for existing hasura cli file stat: - path: "{{ api_hasura_cli_bin }}" + path: "{{ api_hasura_cli_bin }}" register: api_cli_check # only download new version of api cli, when not restoring from backup: -- name: download {{ api_hasura_version }} hasura cli binary +- name: Get Hasura release info from GitHub (authenticated) + uri: + url: "https://api.github.com/repos/hasura/graphql-engine/releases/tags/{{ api_hasura_version }}" + method: GET + headers: + Accept: "application/vnd.github+json" + return_content: true + register: hasura_release + environment: "{{ proxy_env }}" + when: not api_cli_check.stat.exists + +- name: Extract Hasura CLI asset id for {{ linux_architecture }} + set_fact: + hasura_cli_asset_id: >- + {{ + hasura_release.json.assets + | selectattr('name', 'equalto', 'cli-hasura-linux-' ~ linux_architecture) + | map(attribute='id') + | list + | first + }} + when: not api_cli_check.stat.exists + +- name: download {{ api_hasura_version }} hasura cli binary via authenticated GitHub access get_url: - url: "https://github.com/hasura/graphql-engine/releases/download/{{ api_hasura_version }}/cli-hasura-linux-amd64" - dest: "{{ api_hasura_cli_bin }}" - force: true - mode: "0755" - owner: "{{ fworch_user }}" - group: "{{ fworch_group }}" + url: "https://api.github.com/repos/hasura/graphql-engine/releases/assets/{{ hasura_cli_asset_id }}" + dest: "{{ api_hasura_cli_bin }}" + headers: + Accept: "application/octet-stream" + force: true + mode: "0755" + owner: "{{ fworch_user }}" + group: "{{ fworch_group }}" environment: "{{ proxy_env }}" become: true - when: not api_cli_check.stat.exists + when: + - not api_cli_check.stat.exists + - hasura_cli_asset_id is defined - name: initialize hasura cli directory command: "{{ api_hasura_cli_bin }} init {{ product_name }} --skip-update-check --endpoint http://{{ api_local_listening_ip_address }}:{{ api_port }} --admin-secret {{ api_hasura_admin_secret }}" - args: - chdir: "{{ api_home }}" + args: + chdir: "{{ api_home }}" become: true become_user: "{{ fworch_user }}" environment: "{{ proxy_env }}" @@ -87,58 +114,55 @@ - name: set hasura env variable set_fact: - hasura_env: - HASURA_GRAPHQL_DATABASE_URL: "postgres://{{ api_user }}:{{ api_user_password }}@{{ fworch_db_host }}:{{ fworch_db_port }}/{{ fworch_db_name }}" - HASURA_GRAPHQL_ENABLE_CONSOLE: "true" - HASURA_GRAPHQL_ENABLE_TELEMETRY: "false" - HASURA_GRAPHQL_ADMIN_SECRET: "{{ api_hasura_admin_secret }}" - HASURA_GRAPHQL_SERVER_HOST: "127.0.0.1" - HASURA_GRAPHQL_SERVER_PORT: "8080" - HASURA_GRAPHQL_LOG_LEVEL: "{{ api_log_level }}" - HASURA_GRAPHQL_ENABLED_LOG_TYPES: '{{ api_HASURA_GRAPHQL_ENABLED_LOG_TYPES }}' - HASURA_GRAPHQL_CONSOLE_ASSETS_DIR: "/srv/console-assets" - HASURA_GRAPHQL_V1_BOOLEAN_NULL_COLLAPSE: "true" - HASURA_GRAPHQL_CORS_DOMAIN: "*" - HASURA_GRAPHQL_INFER_FUNCTION_PERMISSIONS: "{{ api_HASURA_GRAPHQL_INFER_FUNCTION_PERMISSIONS }}" - HASURA_GRAPHQL_JWT_SECRET: ' - { - "type": "{{ api_hasura_jwt_alg|quote }}", - "key": "{{ api_hasura_jwt_secret | regex_replace(''\n'', ''\\n'') }}", - "claims_namespace_path": "$" - } - ' - HTTP_PROXY: "{{ http_proxy }}" - HTTPS_PROXY: "{{ https_proxy }}" - http_proxy: "{{ http_proxy }}" - https_proxy: "{{ https_proxy }}" - no_proxy: "{{ no_proxy }}" - NO_PROXY: "{{ no_proxy }}" + hasura_env: + HASURA_GRAPHQL_DATABASE_URL: "postgres://{{ api_user }}:{{ api_user_password }}@{{ fworch_db_host }}:{{ fworch_db_port }}/{{ fworch_db_name }}" + HASURA_GRAPHQL_ENABLE_CONSOLE: "true" + HASURA_GRAPHQL_ENABLE_TELEMETRY: "false" + HASURA_GRAPHQL_ADMIN_SECRET: "{{ api_hasura_admin_secret }}" + HASURA_GRAPHQL_SERVER_HOST: "127.0.0.1" + HASURA_GRAPHQL_SERVER_PORT: "8080" + HASURA_GRAPHQL_LOG_LEVEL: "{{ api_log_level }}" + HASURA_GRAPHQL_ENABLED_LOG_TYPES: "{{ api_HASURA_GRAPHQL_ENABLED_LOG_TYPES }}" + HASURA_GRAPHQL_CONSOLE_ASSETS_DIR: "/srv/console-assets" + HASURA_GRAPHQL_V1_BOOLEAN_NULL_COLLAPSE: "true" + HASURA_GRAPHQL_CORS_DOMAIN: "*" + HASURA_GRAPHQL_INFER_FUNCTION_PERMISSIONS: "{{ api_HASURA_GRAPHQL_INFER_FUNCTION_PERMISSIONS }}" + HASURA_GRAPHQL_JWT_SECRET: ' + { + "type": "{{ api_hasura_jwt_alg|quote }}", + "key": "{{ api_hasura_jwt_secret | regex_replace(''\n'', ''\\n'') }}", + "claims_namespace_path": "$" + } + ' + HTTP_PROXY: "{{ http_proxy }}" + HTTPS_PROXY: "{{ https_proxy }}" + http_proxy: "{{ http_proxy }}" + https_proxy: "{{ https_proxy }}" + no_proxy: "{{ no_proxy }}" + NO_PROXY: "{{ no_proxy }}" - name: show hasura env for debugging debug: - var: - hasura_env + var: hasura_env when: debug_level > '1' - name: start hasura container docker_container: - name: "{{ api_container_name }}" - image: hasura/graphql-engine:{{ api_hasura_version }} - state: started - network_mode: host - networks_cli_compatible: true - log_driver: syslog - log_options: - syslog-address: "{{ syslog_proto }}://{{ syslog_host }}:{{ syslog_port }}" - syslog-facility: daemon - tag: "{{ api_container_name }}" - recreate: true - exposed_ports: - - "{{ api_port }}:{{ api_port }}" - env: - "{{ hasura_env }}" - container_default_behavior: no_defaults - user: "1001:1001" # hasura user and group id + name: "{{ api_container_name }}" + image: hasura/graphql-engine:{{ api_hasura_version }} + state: started + network_mode: host + networks_cli_compatible: true + log_driver: syslog + log_options: + syslog-address: "{{ syslog_proto }}://{{ syslog_host }}:{{ syslog_port }}" + syslog-facility: daemon + tag: "{{ api_container_name }}" + recreate: true + env: "{{ hasura_env }}" + container_default_behavior: no_defaults + user: "1001:1001" # hasura user and group id + pull: no register: docker_return become: true become_user: "{{ fworch_user }}" @@ -146,65 +170,64 @@ - name: show docker result debug: - var: - docker_return + var: docker_return when: debug_level > '1' - name: Get info on container docker_container_info: - name: "{{ api_container_name }}" + name: "{{ api_container_name }}" register: result become: true become_user: "{{ fworch_user }}" - name: Print the status of the container in case of problems only - fail: - msg: "The container status is: {{ result }}" + fail: + msg: "The container status is: {{ result }}" when: result.exists and result.container['State']['Status'] == 'exited' - name: copy hasura systemd service script template: - src: "{{ api_service_name }}.service.j2" - dest: "/lib/systemd/system/{{ api_service_name }}.service" - backup: true - mode: "0644" - owner: "root" + src: "{{ api_service_name }}.service.j2" + dest: "/lib/systemd/system/{{ api_service_name }}.service" + backup: true + mode: "0644" + owner: "root" become: true - name: make hasura docker container run at host startup systemd: - name: "{{ api_service_name }}" - daemon_reload: true - enabled: true + name: "{{ api_service_name }}" + daemon_reload: true + enabled: true become: true -- name: wait for hasura port to become available +- name: wait for hasura port to become available wait_for: - port: "{{ api_port }}" - host: "{{ api_local_listening_ip_address }}" - connect_timeout: 1 - delay: 10 - timeout: 25 + port: "{{ api_port }}" + host: "{{ api_local_listening_ip_address }}" + connect_timeout: 1 + delay: 10 + timeout: 25 - name: check for existing api dir from restore stat: - path: "{{ api_home }}/{{ product_name }}" + path: "{{ api_home }}/{{ product_name }}" register: api_metadata_check - name: import API metadata via metadata API directly from local file - uri: - url: "http://{{ api_local_listening_ip_address }}:{{ api_port }}/v1/metadata" - method: POST - return_content: true - body_format: json - headers: - Content-Type: application/json - x-hasura-admin-secret: "{{ api_hasura_admin_secret }}" - x-hasura-role: "admin" - body: "{{ lookup('file','replace_metadata.json') | from_json }}" + uri: + url: "http://{{ api_local_listening_ip_address }}:{{ api_port }}/v1/metadata" + method: POST + return_content: true + body_format: json + headers: + Content-Type: application/json + x-hasura-admin-secret: "{{ api_hasura_admin_secret }}" + x-hasura-role: "admin" + body: "{{ lookup('file','replace_metadata.json') | from_json }}" when: not api_rollback_is_running | bool # do not install latest metadata in case of rollback environment: - http_proxy: "" - https_proxy: "" - # do not use http proxy for metadata import + http_proxy: "" + https_proxy: "" + # do not use http proxy for metadata import diff --git a/roles/common/files/maintenance-info.html b/roles/common/files/maintenance-info.html index 4a8362790a..f31b9ebf93 100644 --- a/roles/common/files/maintenance-info.html +++ b/roles/common/files/maintenance-info.html @@ -33,7 +33,7 @@

Firewall Orchestrator is under maintenance

Sorry for the inconvenience. Please try again in 10 minutes.

- Maintenance Image + Maintenance Image
diff --git a/roles/common/files/men-at-work.jpg b/roles/common/files/men-at-work.jpg deleted file mode 100644 index b3fa379980..0000000000 Binary files a/roles/common/files/men-at-work.jpg and /dev/null differ diff --git a/roles/common/files/men-at-work.png b/roles/common/files/men-at-work.png new file mode 100644 index 0000000000..4885360cba Binary files /dev/null and b/roles/common/files/men-at-work.png differ diff --git a/roles/common/tasks/maintenance-site.yml b/roles/common/tasks/maintenance-site.yml index b65f87e672..b72d84a8e1 100644 --- a/roles/common/tasks/maintenance-site.yml +++ b/roles/common/tasks/maintenance-site.yml @@ -16,8 +16,8 @@ - name: copy maintenance web site image copy: - src: men-at-work.jpg - dest: "/var/www/html/men-at-work.jpg" + src: men-at-work.png + dest: "/var/www/html/men-at-work.png" mode: "0644" - name: enable apache2 required modules diff --git a/roles/common/templates/httpd-maintenance.conf b/roles/common/templates/httpd-maintenance.conf index 340abb986a..3170c7fe0f 100644 --- a/roles/common/templates/httpd-maintenance.conf +++ b/roles/common/templates/httpd-maintenance.conf @@ -7,7 +7,7 @@ RewriteEngine On RewriteCond %{HTTP_HOST} ^(.*)$ RewriteCond %{REQUEST_URI} !^/$ - RewriteCond %{REQUEST_URI} !^/men-at-work.jpg$ + RewriteCond %{REQUEST_URI} !^/men-at-work.png$ RewriteRule ^(.*)$ / [R=301,L] ErrorLog /var/log/{{ webserver_package_name }}/error.log @@ -33,7 +33,7 @@ RewriteEngine On RewriteCond %{HTTP_HOST} ^(.*)$ RewriteCond %{REQUEST_URI} !^/$ - RewriteCond %{REQUEST_URI} !^/men-at-work.jpg$ + RewriteCond %{REQUEST_URI} !^/men-at-work.png$ RewriteRule ^(.*)$ / [R=301,L] # --- Security Headers --- diff --git a/roles/database/files/sql/idempotent/fworch-rule-recert.sql b/roles/database/files/sql/idempotent/fworch-rule-recert.sql index 53e1b16c59..c6dc84d2fe 100644 --- a/roles/database/files/sql/idempotent/fworch-rule-recert.sql +++ b/roles/database/files/sql/idempotent/fworch-rule-recert.sql @@ -8,166 +8,62 @@ - --- fundamental function to check owner <--> rule mapping using the existing view --- "view_rule_with_owner" -CREATE OR REPLACE FUNCTION recert_owner_responsible_for_rule (i_owner_id INTEGER, i_rule_id BIGINT) RETURNS BOOLEAN AS $$ +-- This function returns a table of future recert entries +-- but does not write them into the recertification table +CREATE OR REPLACE FUNCTION recert_get_one_owner_one_mgm( + i_owner_id INTEGER, + i_mgm_id INTEGER +) +RETURNS SETOF recertification AS +$$ DECLARE - i_id BIGINT; + b_super_owner BOOLEAN := FALSE; BEGIN - -- check if this is the super owner: - SELECT INTO i_id id FROM owner WHERE id=i_owner_id AND is_default; - IF FOUND THEN -- this is the super owner - SELECT INTO i_id rule_id FROM view_rule_with_owner WHERE owner_id IS NULL AND rule_id=i_rule_id; - IF FOUND THEN - RAISE DEBUG '%', 'rule found for super owner ' || i_rule_id; - RETURN TRUE; - ELSE - RETURN FALSE; - END IF; - ELSE -- standard owner - SELECT INTO i_id rule_id FROM view_rule_with_owner WHERE owner_id=i_owner_id AND rule_id=i_rule_id; - IF FOUND THEN - RETURN TRUE; - ELSE - RETURN FALSE; - END IF; - END IF; + -- Check if this is the super owner + SELECT TRUE INTO b_super_owner FROM owner WHERE id = i_owner_id AND is_default; + + RETURN QUERY + SELECT DISTINCT + NULL::bigint AS id, + M.rule_metadata_id, + R.rule_id, + V.matches::VARCHAR AS ip_match, + CASE WHEN b_super_owner THEN NULL ELSE i_owner_id END AS owner_id, + NULL::VARCHAR AS user_dn, + FALSE::BOOLEAN AS recertified, + NULL::TIMESTAMP AS recert_date, + NULL::VARCHAR AS comment, + MAX(( + SELECT MAX(value)::TIMESTAMP + FROM ( + SELECT I.start_time::timestamp + make_interval(days => O.recert_interval) AS value + UNION + SELECT C.recert_date + make_interval(days => O.recert_interval) AS value + ) AS tmp + )) AS next_recert_date, + NULL::bigint AS owner_recert_id + FROM + view_rule_with_owner V + LEFT JOIN rule R USING (rule_id) + LEFT JOIN rule_metadata M ON (R.rule_uid = M.rule_uid AND R.dev_id = M.dev_id) + LEFT JOIN owner O ON ( + CASE WHEN b_super_owner THEN O.is_default ELSE V.owner_id = O.id END + ) + LEFT JOIN import_control I ON (R.rule_create = I.control_id) + LEFT JOIN recertification C ON (M.rule_metadata_id = C.rule_metadata_id) + WHERE + ( + (b_super_owner AND V.owner_id IS NULL) + OR + (NOT b_super_owner AND V.owner_id = i_owner_id) + ) + AND R.mgm_id = i_mgm_id + AND R.active + AND (recert_date IS NULL OR (recert_date IS NOT NULL AND recertified)) + GROUP BY M.rule_metadata_id, R.rule_id, V.matches; END; -$$ LANGUAGE plpgsql; - --- this function deletes existing (future) open recert entries and inserts the new ones into the recertificaiton table --- the new recert date will only replace an existing one, if it is closer (smaller) -CREATE OR REPLACE FUNCTION recert_refresh_one_owner_one_mgm - (i_owner_id INTEGER, i_mgm_id INTEGER, t_requested_next_recert_date TIMESTAMP) RETURNS VOID AS $$ -DECLARE - r_rule RECORD; - i_recert_entry_id BIGINT; - b_super_owner BOOLEAN := FALSE; - t_rule_created TIMESTAMP; - t_current_next_recert_date TIMESTAMP; - t_next_recert_date_by_interval TIMESTAMP; - t_rule_last_recertified TIMESTAMP; - t_next_recert_date TIMESTAMP; - i_recert_inverval INTEGER; - b_never_recertified BOOLEAN := FALSE; - b_no_current_next_recert_date BOOLEAN := FALSE; - b_super_owner_exists BOOLEAN := FALSE; - i_previous_import BIGINT; - i_current_import_id BIGINT; - i_super_owner_id INT; - i_current_owner_id_tmp INT; -BEGIN - IF i_owner_id IS NULL OR i_mgm_id IS NULL THEN - IF i_owner_id IS NULL THEN - RAISE WARNING 'found undefined owner_id in recert_refresh_one_owner_one_mgm'; - ELSE -- mgm_id NULL - RAISE WARNING 'found undefined mgm_id in recert_refresh_one_owner_one_mgm'; - END IF; - ELSE - -- get id of previous import: - SELECT INTO i_current_import_id control_id FROM import_control WHERE mgm_id=i_mgm_id AND stop_time IS NULL; - SELECT INTO i_previous_import * FROM get_previous_import_id_for_mgmt(i_mgm_id,i_current_import_id); - IF NOT FOUND OR i_previous_import IS NULL THEN - i_previous_import := -1; -- prevent match for previous import - END IF; - - SELECT INTO i_super_owner_id id FROM owner WHERE is_default; - IF FOUND THEN - b_super_owner_exists := TRUE; - END IF; - - SELECT INTO i_current_owner_id_tmp id FROM owner WHERE id=i_owner_id AND is_default; - IF FOUND THEN - b_super_owner := TRUE; - END IF; - - SELECT INTO i_recert_inverval recert_interval FROM owner WHERE id=i_owner_id; - - FOR r_rule IN - SELECT rule_uid, rule_id FROM rule WHERE mgm_id=i_mgm_id AND (active OR NOT active AND rule_last_seen=i_previous_import) - LOOP - - IF recert_owner_responsible_for_rule (i_owner_id, r_rule.rule_id) THEN - - -- collects dates - SELECT INTO t_current_next_recert_date next_recert_date FROM recertification - WHERE owner_id=i_owner_id AND rule_id=r_rule.rule_id AND recert_date IS NULL; - - IF NOT FOUND THEN - b_no_current_next_recert_date := TRUE; - END IF; - - SELECT INTO t_rule_last_recertified MAX(recert_date) - FROM recertification - WHERE rule_id=r_rule.rule_id AND NOT recert_date IS NULL; - - IF NOT FOUND OR t_rule_last_recertified IS NULL THEN -- no prior recertification, use initial rule import date - b_never_recertified := TRUE; - SELECT INTO t_rule_created rule_metadata.rule_created - FROM rule - LEFT JOIN rule_metadata ON (rule.rule_uid=rule_metadata.rule_uid AND rule.dev_id=rule_metadata.dev_id) - WHERE rule_id=r_rule.rule_id; - END IF; - - IF t_requested_next_recert_date IS NULL THEN - -- if the currenct next recert date is before the intended fixed input date, ignore it - IF b_never_recertified THEN - t_next_recert_date := t_rule_created + make_interval (days => i_recert_inverval); - ELSE - t_next_recert_date := t_rule_last_recertified + make_interval (days => i_recert_inverval); - END IF; - ELSE - t_next_recert_date := t_requested_next_recert_date; - END IF; - - -- do not set next recert date later than actually calculated date - IF NOT b_no_current_next_recert_date THEN - IF t_next_recert_date>t_current_next_recert_date THEN - t_next_recert_date := t_current_next_recert_date; - END IF; - END IF; - - -- delete old recert entry: - DELETE FROM recertification WHERE owner_id=i_owner_id AND rule_id=r_rule.rule_id AND recert_date IS NULL; - - -- add new recert entry: - IF b_super_owner THEN -- special case for super owner (convert NULL to ID) - INSERT INTO recertification (rule_metadata_id, next_recert_date, rule_id, ip_match, owner_id) - SELECT rule_metadata_id, - t_next_recert_date AS next_recert_date, - rule_id, - matches as ip_match, - i_owner_id AS owner_id - FROM view_rule_with_owner - LEFT JOIN rule USING (rule_id) - LEFT JOIN rule_metadata ON (rule.rule_uid=rule_metadata.rule_uid AND rule.dev_id=rule_metadata.dev_id) - WHERE view_rule_with_owner.rule_id=r_rule.rule_id AND view_rule_with_owner.owner_id IS NULL; - ELSE - INSERT INTO recertification (rule_metadata_id, next_recert_date, rule_id, ip_match, owner_id) - SELECT rule_metadata_id, - t_next_recert_date AS next_recert_date, - rule_id, - matches as ip_match, - i_owner_id AS owner_id - FROM view_rule_with_owner - LEFT JOIN rule USING (rule_id) - LEFT JOIN rule_metadata ON (rule.rule_uid=rule_metadata.rule_uid AND rule.dev_id=rule_metadata.dev_id) - WHERE view_rule_with_owner.rule_id=r_rule.rule_id AND view_rule_with_owner.owner_id=i_owner_id; - END IF; - ELSE - -- delete old outdated recert entry if owner is not responsible any more - DELETE FROM recertification WHERE owner_id=i_owner_id AND rule_id=r_rule.rule_id AND recert_date IS NULL; - END IF; - END LOOP; +$$ LANGUAGE plpgsql STABLE; - -- -- finally, when not super user - recalculate super user recert entries - since these might change with each owner change - -- IF NOT b_super_owner AND b_super_owner_exists THEN - -- PERFORM recert_refresh_one_owner_one_mgm (i_super_owner_id, i_mgm_id, t_requested_next_recert_date); - -- END IF; - END IF; -END; -$$ LANGUAGE plpgsql; -- function used during import of a single management config @@ -188,6 +84,7 @@ BEGIN END; $$ LANGUAGE plpgsql; +-- select * from recert_get_one_owner_one_mgm(4,1) -- this function returns a table of future recert entries -- but does not write them into the recertification table @@ -226,7 +123,8 @@ BEGIN SELECT I.start_time::timestamp + make_interval (days => o.recert_interval) AS value UNION SELECT C.recert_date + make_interval (days => o.recert_interval) AS value - ) AS temp_table)) + ) AS temp_table)), + NULL::bigint AS owner_recert_id FROM view_rule_with_owner V LEFT JOIN rule R USING (rule_id) @@ -253,7 +151,8 @@ BEGIN SELECT I.start_time::timestamp + make_interval (days => o.recert_interval) AS value UNION SELECT C.recert_date + make_interval (days => o.recert_interval) AS value - ) AS temp_table)) + ) AS temp_table)), + NULL::bigint AS owner_recert_id FROM view_rule_with_owner V LEFT JOIN rule R USING (rule_id) diff --git a/roles/lib/files/FWO.Api.Client/APIcalls/recertification/getOpenRecerts.graphql b/roles/lib/files/FWO.Api.Client/APIcalls/recertification/getOpenRecerts.graphql deleted file mode 100644 index 0c96596aa8..0000000000 --- a/roles/lib/files/FWO.Api.Client/APIcalls/recertification/getOpenRecerts.graphql +++ /dev/null @@ -1,17 +0,0 @@ -query getFutureRecertsForOwners($ownerId: Int!, $mgmId: Int!) { - recert_get_one_owner_one_mgm( - where: { recert_date: { _is_null: true } } - args: { i_mgm_id: $mgmId, i_owner_id: $ownerId } - ) { - id - rule_metadata_id - rule_id - ip_match - owner_id - user_dn - recertified - next_recert_date - recert_date - comment - } -} diff --git a/roles/lib/files/FWO.Api.Client/APIcalls/recertification/getOpenRecertsForOwners.graphql b/roles/lib/files/FWO.Api.Client/APIcalls/recertification/getOpenRecertsForOwners.graphql new file mode 100644 index 0000000000..47357e44ab --- /dev/null +++ b/roles/lib/files/FWO.Api.Client/APIcalls/recertification/getOpenRecertsForOwners.graphql @@ -0,0 +1,17 @@ +query getOpenRecertsForOwners($ownerId: Int!, $mgmId: Int!) { + recert_get_one_owner_one_mgm( + where: { recert_date: { _is_null: true } } + args: { i_mgm_id: $mgmId, i_owner_id: $ownerId } + ) { + id + rule_metadata_id + rule_id + ip_match + owner_id + user_dn + recertified + next_recert_date + recert_date + comment + } +} diff --git a/roles/lib/files/FWO.Api.Client/Queries/RecertQueries.cs b/roles/lib/files/FWO.Api.Client/Queries/RecertQueries.cs index 6788df8364..87b85ebc63 100644 --- a/roles/lib/files/FWO.Api.Client/Queries/RecertQueries.cs +++ b/roles/lib/files/FWO.Api.Client/Queries/RecertQueries.cs @@ -12,7 +12,7 @@ public class RecertQueries : Queries public static readonly string recertifyOwner; public static readonly string recertifyRuleDirectly; public static readonly string getOpenRecertsForRule; - public static readonly string getOpenRecerts; + public static readonly string getOpenRecertsForOwners; public static readonly string clearOpenRecerts; public static readonly string addRecertEntries; public static readonly string refreshViewRuleWithOwner; @@ -36,7 +36,7 @@ static RecertQueries() recertifyOwner = File.ReadAllText(QueryPath + "recertification/recertifyOwner.graphql"); recertifyRuleDirectly = File.ReadAllText(QueryPath + "recertification/recertifyRuleDirectly.graphql"); getOpenRecertsForRule = File.ReadAllText(QueryPath + "recertification/getOpenRecertsForRule.graphql"); - getOpenRecerts = File.ReadAllText(QueryPath + "recertification/getOpenRecerts.graphql"); + getOpenRecertsForOwners = File.ReadAllText(QueryPath + "recertification/getOpenRecertsForOwners.graphql"); clearOpenRecerts = File.ReadAllText(QueryPath + "recertification/clearOpenRecerts.graphql"); addRecertEntries = File.ReadAllText(QueryPath + "recertification/addRecertEntries.graphql"); refreshViewRuleWithOwner = File.ReadAllText(QueryPath + "recertification/refreshViewRuleWithOwner.graphql"); diff --git a/roles/lib/files/FWO.Basics/FWO.Basics.csproj b/roles/lib/files/FWO.Basics/FWO.Basics.csproj index 09961d5a36..71dc1bb332 100644 --- a/roles/lib/files/FWO.Basics/FWO.Basics.csproj +++ b/roles/lib/files/FWO.Basics/FWO.Basics.csproj @@ -7,6 +7,7 @@ + diff --git a/roles/lib/files/FWO.Basics/IpOperations.cs b/roles/lib/files/FWO.Basics/IpOperations.cs index 97747301b0..ff6aa505a1 100644 --- a/roles/lib/files/FWO.Basics/IpOperations.cs +++ b/roles/lib/files/FWO.Basics/IpOperations.cs @@ -2,18 +2,56 @@ using System.Net; using NetTools; using System; +using DnsClient; namespace FWO.Basics { public static class IpOperations { + + // reuse the client to avoid socket churn; disable client-side cache + private static readonly LookupClient ReverseLookupClient = new(new LookupClientOptions + { + UseCache = false, + ContinueOnDnsError = true, + ThrowDnsErrors = false + }); + + public static async Task> DnsReverseLookUpAllAsync( + IPAddress address, + CancellationToken cancellationToken = default) + { + // QueryReverseAsync issues a PTR query and returns all answers from the DNS server + IDnsQueryResponse response = await ReverseLookupClient.QueryReverseAsync(address, cancellationToken) + .ConfigureAwait(false); + + if (response.HasError || response.Answers.Count == 0) + { + return Array.Empty(); + } + + return response.Answers + .PtrRecords() + .Select(ptr => ptr.PtrDomainName.Value.TrimEnd('.')) // drop trailing dot + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + public static async Task DnsReverseLookUpPreferredAsync(IPAddress address) + { + IReadOnlyList names = await DnsReverseLookUpAllAsync(address); + return names.FirstOrDefault(name => !name.StartsWith("lx", StringComparison.OrdinalIgnoreCase)) + ?? names.FirstOrDefault() + ?? string.Empty; + } + public static async Task DnsReverseLookUp(IPAddress address) { try { return (await Dns.GetHostEntryAsync(address)).HostName; } - catch(Exception) + catch (Exception) { return ""; } @@ -51,25 +89,25 @@ public static bool TryParseIPStringToRange(this string ipString, out (string Sta bool ipStartOK = IPAddress.TryParse(ipStart, out IPAddress? ipAdressStart); bool ipEndOK = IPAddress.TryParse(ipEnd, out IPAddress? ipAdressEnd); - if(ipAdressStart is null || ipAdressEnd is null) + if (ipAdressStart is null || ipAdressEnd is null) { return false; } - if(strictv4Parse && ipAdressStart?.AddressFamily == AddressFamily.InterNetwork && ipAdressEnd?.AddressFamily == AddressFamily.InterNetwork) + if (strictv4Parse && ipAdressStart?.AddressFamily == AddressFamily.InterNetwork && ipAdressEnd?.AddressFamily == AddressFamily.InterNetwork) { - if(!IsValidIPv4(ipStart) || !IsValidIPv4(ipEnd)) + if (!IsValidIPv4(ipStart) || !IsValidIPv4(ipEnd)) { return false; } } - if(!ipStartOK || !ipEndOK) + if (!ipStartOK || !ipEndOK) { return false; } - if(!IPAddress.TryParse(ipStart, out _) || !IPAddress.TryParse(ipEnd, out _)) + if (!IPAddress.TryParse(ipStart, out _) || !IPAddress.TryParse(ipEnd, out _)) { return false; } @@ -79,16 +117,16 @@ public static bool TryParseIPStringToRange(this string ipString, out (string Sta return true; } - catch(Exception) + catch (Exception) { return false; } } - public static bool TryParseIPString(this string ipString, out T? ipResult, bool strictv4Parse = false) + public static bool TryParseIPString(this string ipString, out T? ipResult, bool strictv4Parse = false) { ipResult = default; - + try { (string ipStart, string ipEnd) = SplitIpToRange(ipString); @@ -96,34 +134,35 @@ public static bool TryParseIPString(this string ipString, out T? ipResult, bo bool ipStartOK = IPAddress.TryParse(ipStart, out IPAddress? ipAdressStart); bool ipEndOK = IPAddress.TryParse(ipEnd, out IPAddress? ipAdressEnd); - if(ipAdressStart is null || ipAdressEnd is null) + if (ipAdressStart is null || ipAdressEnd is null) { return false; } - if(strictv4Parse && ipAdressStart?.AddressFamily == AddressFamily.InterNetwork && ipAdressEnd?.AddressFamily == AddressFamily.InterNetwork) + if (strictv4Parse && ipAdressStart?.AddressFamily == AddressFamily.InterNetwork && ipAdressEnd?.AddressFamily == AddressFamily.InterNetwork) { - if(!IsValidIPv4(ipStart) || !IsValidIPv4(ipEnd)) + if (!IsValidIPv4(ipStart) || !IsValidIPv4(ipEnd)) { return false; } } - if(!ipStartOK || !ipEndOK) + if (!ipStartOK || !ipEndOK) { return false; } - if(typeof(T) == typeof((string, string))) + if (typeof(T) == typeof((string, string))) { ipResult = (T)Convert.ChangeType((ipAdressStart!.ToString(), ipAdressEnd!.ToString()), typeof(T)); return true; } - else if(typeof(T) == typeof(IPAddressRange) && IPAddressRange.TryParse(ipString, out IPAddressRange ipRange)) + else if (typeof(T) == typeof(IPAddressRange) && IPAddressRange.TryParse(ipString, out IPAddressRange ipRange)) { ipResult = (T)Convert.ChangeType(ipRange, typeof(T)); return true; - }else if(typeof(T) == typeof((IPAddress, IPAddress))) + } + else if (typeof(T) == typeof((IPAddress, IPAddress))) { Tuple? ipTuple = new(ipAdressStart!, ipAdressEnd!); ipResult = (T)Convert.ChangeType(ipTuple, typeof(T)); @@ -132,7 +171,7 @@ public static bool TryParseIPString(this string ipString, out T? ipResult, bo return false; } - catch(Exception) + catch (Exception) { return false; } @@ -142,7 +181,7 @@ private static bool IsValidIPv4(string ipAddress) { byte[] addBytes = [.. ipAddress.Split('.').Where(_ => byte.Parse(_) <= 255 && byte.Parse(_) >= 0).Select(byte.Parse)]; - return addBytes.Length == 4; + return addBytes.Length == 4; } public static string GetObjectType(string ip1, string ip2) @@ -333,11 +372,11 @@ public static int CompareIpValues(IPAddress ip1, IPAddress ip2) /// public static int CompareIpFamilies(IPAddress ip1, IPAddress ip2) { - if (ip1.AddressFamily == AddressFamily.InterNetwork && ip2.AddressFamily == AddressFamily.InterNetworkV6 ) + if (ip1.AddressFamily == AddressFamily.InterNetwork && ip2.AddressFamily == AddressFamily.InterNetworkV6) { return -1; } - if (ip1.AddressFamily == AddressFamily.InterNetworkV6 && ip2.AddressFamily == AddressFamily.InterNetwork ) + if (ip1.AddressFamily == AddressFamily.InterNetworkV6 && ip2.AddressFamily == AddressFamily.InterNetwork) { return 1; } diff --git a/roles/lib/files/FWO.Recert/RecertHandler.cs b/roles/lib/files/FWO.Recert/RecertHandler.cs index 025ad48c4f..50c61acee7 100644 --- a/roles/lib/files/FWO.Recert/RecertHandler.cs +++ b/roles/lib/files/FWO.Recert/RecertHandler.cs @@ -27,7 +27,7 @@ public async Task RecertifySingleRule(Rule rule, FwoOwner? owner, string? recertified = rule.Metadata.Recert, recertDate = DateTime.Now, comment = comment, - ownerRecertId = owner?.LastRecertId + ownerRecertId = owner?.LastRecertId == 0 ? null : owner?.LastRecertId }; bool recertOk = (await apiConnection.SendQueryAsync(RecertQueries.recertify, variables)).AffectedRows > 0; if (rule.Metadata.Recert) diff --git a/roles/lib/files/FWO.Recert/RecertRefresh.cs b/roles/lib/files/FWO.Recert/RecertRefresh.cs index 969a3d3dd2..795836d32b 100644 --- a/roles/lib/files/FWO.Recert/RecertRefresh.cs +++ b/roles/lib/files/FWO.Recert/RecertRefresh.cs @@ -10,7 +10,7 @@ public static class RecertRefresh { public static async Task RecalcRecerts(ApiConnection apiConnection) { - Stopwatch watch = new (); + Stopwatch watch = new(); try { @@ -42,13 +42,13 @@ public static async Task RecalcRecerts(ApiConnection apiConnection) private static async Task RecalcRecertsOfOwner(FwoOwner owner, List managements, ApiConnection apiConnection) { - Stopwatch watch = new (); + Stopwatch watch = new(); watch.Start(); - + foreach (Management mgm in managements) { List currentRecerts = - await apiConnection.SendQueryAsync>(RecertQueries.getOpenRecerts, new { ownerId = owner.Id, mgmId = mgm.Id }); + await apiConnection.SendQueryAsync>(RecertQueries.getOpenRecertsForOwners, new { ownerId = owner.Id, mgmId = mgm.Id }); if (currentRecerts.Count > 0) { diff --git a/roles/lib/files/FWO.Services/AppServerHelper.cs b/roles/lib/files/FWO.Services/AppServerHelper.cs index 36a5cfb327..8b9de7c2e4 100644 --- a/roles/lib/files/FWO.Services/AppServerHelper.cs +++ b/roles/lib/files/FWO.Services/AppServerHelper.cs @@ -13,14 +13,14 @@ namespace FWO.Services public static class AppServerHelper { public static async Task ConstructAppServerNameFromDns(ModellingAppServer appServer, ModellingNamingConvention namingConvention, - bool overwriteExistingNames=false, bool logUnresolvable=false) + bool overwriteExistingNames = false, bool logUnresolvable = false) { if ((string.IsNullOrEmpty(appServer.IpEnd) || appServer.IpEnd == appServer.Ip) && IPAddress.TryParse(appServer.Ip.StripOffNetmask(), out IPAddress? ip)) { - string dnsName = await IpOperations.DnsReverseLookUp(ip); - if(string.IsNullOrEmpty(dnsName)) + string dnsName = await IpOperations.DnsReverseLookUpPreferredAsync(ip); + if (string.IsNullOrEmpty(dnsName)) { - if(logUnresolvable) + if (logUnresolvable) { Log.WriteWarning("Import App Server Data", $"Found empty (unresolvable) IP {appServer.Ip}"); } @@ -38,13 +38,13 @@ public static async Task ConstructAppServerNameFromDns(ModellingAppServe return appServer.Name; } - public static string ConstructSanitizedAppServerName(ModellingAppServer appServer, ModellingNamingConvention namingConvention, bool overwriteExistingNames=false) + public static string ConstructSanitizedAppServerName(ModellingAppServer appServer, ModellingNamingConvention namingConvention, bool overwriteExistingNames = false) { bool shortened = false; return ConstructAppServerName(appServer, namingConvention, overwriteExistingNames).SanitizeJsonFieldMand(ref shortened); } - public static string ConstructAppServerName(ModellingAppServer appServer, ModellingNamingConvention namingConvention, bool overwriteExistingNames=false) + public static string ConstructAppServerName(ModellingAppServer appServer, ModellingNamingConvention namingConvention, bool overwriteExistingNames = false) { if (string.IsNullOrEmpty(appServer.Name) || overwriteExistingNames) { @@ -60,10 +60,10 @@ public static async Task AdjustAppServerNames(ApiConnection apiConnection, UserC List AppServers = await apiConnection.SendQueryAsync>(ModellingQueries.getAllAppServers); int correctedCounter = 0; int failCounter = 0; - foreach(var appServer in AppServers) + foreach (var appServer in AppServers) { string oldName = appServer.Name; - if((await ConstructAppServerNameFromDns(appServer, namingConvention, userConfig.OverwriteExistingNames)) != oldName) + if ((await ConstructAppServerNameFromDns(appServer, namingConvention, userConfig.OverwriteExistingNames)) != oldName) { appServer.ImportSource = GlobalConst.kAdjustAppServerNames; if (await UpdateName(apiConnection, userConfig, appServer, oldName)) @@ -86,7 +86,7 @@ public static async Task NoHigherPrioActive(ApiConnection apiConnection, M List ExistingAppServersSameIp = await GetExistingSameIp(apiConnection, incomingAppServer); return ExistingAppServersSameIp.FirstOrDefault(x => Prio(x.ImportSource) > Prio(incomingAppServer.ImportSource) && !x.IsDeleted) == null; } - catch(Exception exception) + catch (Exception exception) { Log.WriteError("Check App Server Prio", $" Check of {incomingAppServer.Name} from {incomingAppServer.ImportSource} to exception:", exception); } @@ -98,7 +98,7 @@ public static async Task ReactivateOtherSource(ApiConnection apiConnection, User try { List ExistingOtherAppServersSameIp = [.. (await GetExistingSameIp(apiConnection, deletedAppServer)).Where(x => x.Id != deletedAppServer.Id)]; - if(ExistingOtherAppServersSameIp != null && ExistingOtherAppServersSameIp.Count > 0) + if (ExistingOtherAppServersSameIp != null && ExistingOtherAppServersSameIp.Count > 0) { int maxPrio = ExistingOtherAppServersSameIp.Max(x => Prio(x.ImportSource)); List ExistingOtherAppServersMaxPrio = [.. ExistingOtherAppServersSameIp.Where(x => Prio(x.ImportSource) == maxPrio)]; @@ -112,13 +112,13 @@ public static async Task ReactivateOtherSource(ApiConnection apiConnection, User await apiConnection.SendQueryAsync(ModellingQueries.setAppServerDeletedState, Variables); await ModellingHandlerBase.LogChange(ModellingTypes.ChangeType.Reactivate, ModellingTypes.ModObjectType.AppServer, reactivatedAppServer.Id, $"Reactivated App Server: {reactivatedAppServer.Display()}", apiConnection, userConfig, reactivatedAppServer.AppId, DefaultInit.DoNothing, null, reactivatedAppServer.ImportSource); - if(userConfig.AutoReplaceAppServer) + if (userConfig.AutoReplaceAppServer) { await ReplaceAppServer(apiConnection, deletedAppServer.Id, reactivatedId); } } } - catch(Exception exception) + catch (Exception exception) { Log.WriteError("Reactivate App Server", $"Reactivation of other than {deletedAppServer.Name} from {deletedAppServer.ImportSource} leads to exception:", exception); } @@ -129,25 +129,25 @@ public static async Task DeactivateOtherSources(ApiConnection apiConnection, Use try { List ExistingActiveAppServersSameIp = [.. (await GetExistingSameIp(apiConnection, incomingAppServer)).Where(x => x.Id != incomingAppServer.Id && !x.IsDeleted)]; - if(ExistingActiveAppServersSameIp != null && ExistingActiveAppServersSameIp.Count > 0) + if (ExistingActiveAppServersSameIp != null && ExistingActiveAppServersSameIp.Count > 0) { - foreach(var activeAppServer in ExistingActiveAppServersSameIp) + foreach (var activeAppServer in ExistingActiveAppServersSameIp) { await DeactivateAppServer(apiConnection, userConfig, activeAppServer, incomingAppServer); } } } - catch(Exception exception) + catch (Exception exception) { Log.WriteError("Deactivate App Servers", $"Deactivation of {incomingAppServer.Name} from {incomingAppServer.ImportSource} leads to exception:", exception); } } - public static async Task<(long?, string?)> UpsertAppServer(ApiConnection apiConnection, UserConfig userConfig, ModellingAppServer incomingAppServer, bool nameCheck, bool manual=false, bool addMode=false) + public static async Task<(long?, string?)> UpsertAppServer(ApiConnection apiConnection, UserConfig userConfig, ModellingAppServer incomingAppServer, bool nameCheck, bool manual = false, bool addMode = false) { try { - if(nameCheck && await CheckNameExisting(apiConnection, incomingAppServer)) + if (nameCheck && await CheckNameExisting(apiConnection, incomingAppServer)) { return (null, incomingAppServer.Name); } @@ -155,7 +155,7 @@ public static async Task DeactivateOtherSources(ApiConnection apiConnection, Use List ExistingAppServersSameIp = await GetExistingSameIp(apiConnection, incomingAppServer); long? AppServerId = null; - if(ExistingAppServersSameIp == null || ExistingAppServersSameIp.Count == 0) + if (ExistingAppServersSameIp == null || ExistingAppServersSameIp.Count == 0) { AppServerId = manual && !addMode ? await UpdateAppServerInDb(apiConnection, userConfig, incomingAppServer) : await AddAppServerToDb(apiConnection, userConfig, incomingAppServer); @@ -171,7 +171,7 @@ public static async Task DeactivateOtherSources(ApiConnection apiConnection, Use if (manual) { ModellingAppServer? otherAppServerSameIp = ExistingAppServersSameIp.FirstOrDefault(x => x.Id != incomingAppServer.Id && !x.IsDeleted); - if(otherAppServerSameIp != null) + if (otherAppServerSameIp != null) { return (null, otherAppServerSameIp.Name); } @@ -179,7 +179,7 @@ public static async Task DeactivateOtherSources(ApiConnection apiConnection, Use return await OverwriteAppServer(apiConnection, userConfig, incomingAppServer, ExistingAppServersSameIp); } - catch(Exception exception) + catch (Exception exception) { Log.WriteError("Upsert App Server", $"Upsert of {incomingAppServer.Name} leads to exception:", exception); return (null, null); @@ -198,7 +198,7 @@ private static async Task> GetExistingSameIp(ApiConnect }; return await apiConnection.SendQueryAsync>(ModellingQueries.getAppServersByIp, Variables); } - catch(Exception exception) + catch (Exception exception) { Log.WriteError("Get Existing App Server", $"leads to exception:", exception); return []; @@ -222,9 +222,9 @@ private static async Task> GetExistingSameIp(ApiConnect AppServerId = await AddAppServerToDb(apiConnection, userConfig, incomingAppServer); } - foreach(var existAppServerOtherSource in existingAppServersSameIp.Where(x => x.ImportSource != incomingAppServer.ImportSource)) + foreach (var existAppServerOtherSource in existingAppServersSameIp.Where(x => x.ImportSource != incomingAppServer.ImportSource)) { - if(!existAppServerOtherSource.IsDeleted) + if (!existAppServerOtherSource.IsDeleted) { await DeactivateAppServer(apiConnection, userConfig, existAppServerOtherSource, incomingAppServer); } @@ -239,12 +239,12 @@ private static async Task DeactivateAppServer(ApiConnection apiConnection, UserC await apiConnection.SendQueryAsync(ModellingQueries.setAppServerDeletedState, new { id = appServerToDeactivate.Id, deleted = true }); await ModellingHandlerBase.LogChange(ModellingTypes.ChangeType.MarkDeleted, ModellingTypes.ModObjectType.AppServer, appServerToDeactivate.Id, $"Deactivated App Server: {appServerToDeactivate.Display()}", apiConnection, userConfig, appServerToDeactivate.AppId, DefaultInit.DoNothing, null, appServerToDeactivate.ImportSource); - if(userConfig.AutoReplaceAppServer) + if (userConfig.AutoReplaceAppServer) { await ReplaceAppServer(apiConnection, appServerToDeactivate.Id, newAppServer.Id); } } - catch(Exception exception) + catch (Exception exception) { Log.WriteError("Deactivate App Server", $"leads to exception:", exception); } @@ -257,7 +257,7 @@ private static async Task ReplaceAppServer(ApiConnection apiConnection, long old await apiConnection.SendQueryAsync(ModellingQueries.updateNwObjectInNwGroup, new { oldObjectId = oldAppServerId, newObjectId = newAppServerId }); await apiConnection.SendQueryAsync(ModellingQueries.updateNwObjectInConnection, new { oldObjectId = oldAppServerId, newObjectId = newAppServerId }); } - catch(Exception exception) + catch (Exception exception) { Log.WriteError("Replace App Server", $"Replacing {oldAppServerId} by {newAppServerId} leads to exception:", exception); } @@ -275,12 +275,12 @@ private static async Task CheckNameExisting(ApiConnection apiConnection, M List ExistingAppServersSameIp = await apiConnection.SendQueryAsync>(ModellingQueries.getAppServersByName, Variables); return ExistingAppServersSameIp != null && ExistingAppServersSameIp.Count > 0 && ExistingAppServersSameIp.FirstOrDefault(x => x.Id == incomingAppServer.Id && !x.IsDeleted) == null; } - catch(Exception exception) + catch (Exception exception) { Log.WriteError("Upsert App Server", $"leads to exception:", exception); return false; } - } + } private static string GetPrefix(ModellingAppServer appServer, ModellingNamingConvention namingConvention) { @@ -292,7 +292,7 @@ private static string GetPrefix(ModellingAppServer appServer, ModellingNamingCon _ => "" }; } - + private static async Task UpdateName(ApiConnection apiConnection, UserConfig userConfig, ModellingAppServer appServer, string oldName) { try @@ -308,7 +308,7 @@ await ModellingHandlerBase.LogChange(ModellingTypes.ChangeType.Update, Modelling Log.WriteDebug($"Correct App Server Name", $"Changed {oldName} to {appServer.Name}."); return true; } - catch(Exception exception) + catch (Exception exception) { Log.WriteError("Correct AppServer Name", $"Leads to exception:", exception); return false; @@ -334,7 +334,7 @@ private static int Prio(string importSource) customType = appServer.CustomType }; ReturnId[]? returnIds = (await apiConnection.SendQueryAsync(ModellingQueries.newAppServer, Variables)).ReturnIds; - if(returnIds != null && returnIds.Length > 0) + if (returnIds != null && returnIds.Length > 0) { appServer.Id = returnIds[0].NewIdLong; await ModellingHandlerBase.LogChange(ModellingTypes.ChangeType.Insert, ModellingTypes.ModObjectType.AppServer, appServer.Id, diff --git a/roles/sample-data/files/config_changes/unused_enlarge_rule.py b/roles/sample-data/files/config_changes/unused_enlarge_rule.py deleted file mode 100644 index e60af3f23a..0000000000 --- a/roles/sample-data/files/config_changes/unused_enlarge_rule.py +++ /dev/null @@ -1,129 +0,0 @@ -# Created by alf -# Enlarges rule 60 by a random srcadress in fortigate.cfg - -import random -import string -import fnmatch -import os -import sys -import re - -# Define global variables that may be passed on the command line and their defaults if not -# example$ python3 enlarge_rule.py uid "path" - -input_index = sys.argv[1] if len(sys.argv) >= 2 else 0 -input_index = int(input_index) % 59 -config_path = sys.argv[2] if len(sys.argv) >= 3 else "/home/fworchsample/sample-configs/fortinet_demo/fortigate.cfg" -uid = -1 - -# First step: create functions that build a random ip address and uuid - - -def random_octet(): - return str(random.randrange(0, 255)) - - -def random_ip(): - return random_octet() + '.' + random_octet() + '.' + random_octet() + '.' + random_octet() - - -def random_uuid(): - s = ''.join(random.choices(string.ascii_lowercase + string.digits, k=32)) - return s[:8] + '-' + s[8:12] + '-' + s[12:16] + '-' + s[16:20] + '-' + s[20:] - - -# Second step: extract rule uid using the input_index - -with open(config_path, "r") as fin: - data = fin.readlines() - -rule_area_flag = False -rule_mapping_count = 0 - -for line in data: - if fnmatch.filter([line], 'config firewall policy\n'): - rule_area_flag = True - elif fnmatch.filter([line], ' edit *\n') and rule_area_flag: - uid = int(re.findall(r'\d+', line)[0]) - if rule_mapping_count == input_index: - break - else: - rule_mapping_count = rule_mapping_count + 1 - elif fnmatch.filter([line], 'end\n') and rule_area_flag: - break - - -# Third step: build new network object in "config firewall address" - -ip_address = random_ip() -uuid = random_uuid() -count = 1 -line_to_insert_at = -1 - -for line in data: - if fnmatch.filter([line], 'config firewall address\n'): - line_to_insert_at = count - break - else: - count = count + 1 - -if line_to_insert_at > -1: - data.insert(line_to_insert_at, '# start recognition comment for auto-delete function, ID={}\n'.format(uid)) - data.insert(line_to_insert_at + 1, ' edit "{}"\n'.format(ip_address)) - data.insert(line_to_insert_at + 2, ' set uuid {}\n'.format(uuid)) - data.insert(line_to_insert_at + 3, ' set subnet {} 255.255.255.255\n'.format(ip_address)) - data.insert(line_to_insert_at + 4, ' set comment "Automatically built for test purposes"\n') - data.insert(line_to_insert_at + 5, ' next\n') - data.insert(line_to_insert_at + 6, '# end recognition comment for auto-delete function, ID={}\n'.format(uid)) - -# Fourth step: add new object to source address of rule uid - -rule_area_flag = False -uid_flag = False -delete_unused_network_objects = False -replace_counter = 0 - -for line in data: - if fnmatch.filter([line], 'config firewall policy\n'): - rule_area_flag = True - if fnmatch.filter([line], ' edit {}\n'.format(uid)): - uid_flag = True - if fnmatch.filter([line], ' set srcaddr*') and rule_area_flag and uid_flag: - if fnmatch.filter([line], ' set srcaddr "all"\n'): - data[replace_counter] = ' set srcaddr "{}"\n'.format(ip_address) - else: - data[replace_counter] = line.rstrip() + ' "{}"\n'.format(ip_address) - if len(data[replace_counter]) > 95: - data[replace_counter] = ' set srcaddr "{}"\n'.format(ip_address) - delete_unused_network_objects = True - break - if fnmatch.filter([line], ' next\n'.format(uid)): - uid_flag = False - if fnmatch.filter([line], 'end\n'): - rule_area_flag = False - replace_counter = replace_counter + 1 - -# Last step: write everything back to the config file -# If delete_unused_network_objects is set delete all new objects except for the most recent - -with open(config_path + ".tmp", "w") as fout: - if delete_unused_network_objects: - delete_flag = False - object_count = 0 - for line in data: - last_comment_line_flag = False - if line == '# start recognition comment for auto-delete function, ID={}\n'.format(uid): - delete_flag = True - object_count = object_count + 1 - if line == '# end recognition comment for auto-delete function, ID={}\n'.format(uid): - delete_flag = False - last_comment_line_flag = True - if object_count < 2: - fout.write(line) - elif not (delete_flag or last_comment_line_flag): - fout.write(line) - else: - for line in data: - fout.write(line) - -os.rename(config_path + ".tmp", config_path) diff --git a/roles/sample-data/files/config_changes/write_date_to_comment.py b/roles/sample-data/files/config_changes/write_date_to_comment.py deleted file mode 100644 index db9e2087d0..0000000000 --- a/roles/sample-data/files/config_changes/write_date_to_comment.py +++ /dev/null @@ -1,43 +0,0 @@ -# Changes the comment in rule x to the current date in fortigate.cfg -# x = 52 by default, can be changed in ansible -# Created by alf - -import fnmatch -import datetime -import os -import sys - -# Define global variables that may be passed on the command line and their defaults if not -# example$ python3 write_date_to_comment.py uid "path" - -uid = sys.argv[1] if len(sys.argv) >= 2 else 52 -config_path = sys.argv[2] if len(sys.argv) >= 3 else "/home/fworchsample/sample-configs/fortinet_demo/fortigate.cfg" - -with open(config_path, "r") as fin: - data = fin.readlines() - -rule_area_flag = False -uid_flag = False -current_line = 0 -for line in data: - if fnmatch.filter([line], 'config firewall policy\n'): - rule_area_flag = True - if fnmatch.filter([line], ' edit {}\n'.format(uid)): - uid_flag = True - if fnmatch.filter([line], ' set comments*') and uid_flag and rule_area_flag: - data[current_line] = ' set comments "{}"\n'.format(datetime.datetime.now()) - break - if fnmatch.filter([line], ' next\n') and uid_flag and rule_area_flag: - data.insert(current_line, ' set comments "{}"\n'.format(datetime.datetime.now())) - break - if fnmatch.filter([line], ' next\n'): - uid_flag = False - if fnmatch.filter([line], 'end\n'): - rule_area_flag = False - current_line = current_line + 1 - -with open(config_path + "2.tmp", "w") as fout: - data = "".join(data) - fout.write(data) - -os.rename(config_path + '2.tmp', config_path) diff --git a/roles/sample-data/files/config_changes/changeRule.py b/roles/tests-integration/files/importer/config_changes/changeRule.py similarity index 94% rename from roles/sample-data/files/config_changes/changeRule.py rename to roles/tests-integration/files/importer/config_changes/changeRule.py index 5fe7fc7f48..4f94c0fb5a 100644 --- a/roles/sample-data/files/config_changes/changeRule.py +++ b/roles/tests-integration/files/importer/config_changes/changeRule.py @@ -9,15 +9,15 @@ import logging -def randomOctet(): +def random_octet(): return str(random.randrange(0, 256)) -def randomIp(): - return randomOctet() + '.' + randomOctet() + '.' + randomOctet() + '.' + randomOctet() +def random_ip(): + return random_octet() + '.' + random_octet() + '.' + random_octet() + '.' + random_octet() -def randomUid(): +def random_uid(): s = ''.join(random.choices(string.ascii_lowercase + string.digits, k=32)) return s[:8] + '-' + s[8:12] + '-' + s[12:16] + '-' + s[16:20] + '-' + s[20:] @@ -76,8 +76,8 @@ def randomUid(): if actionChosen == 'changeSrcOrDst': if not deleteElement: - newUid = randomUid() - newIp = randomIp() + newUid = random_uid() + newIp = random_ip() # cannot add to any obj, so delete it first if anyObj in ruleSide: