Skip to content

Commit 8bd35c1

Browse files
authored
Merge branch 'develop' into 5512-docker-image-size
2 parents 68269c2 + b8ea374 commit 8bd35c1

File tree

24 files changed

+316
-63
lines changed

24 files changed

+316
-63
lines changed

Taskfile.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,9 +91,11 @@ tasks:
9191
backend-exec-seed-db:
9292
desc: Seed DB with initial data in the backend container
9393
dir: tdrs-backend
94+
vars:
95+
SEED_NUM: '{{.SEED_NUM | default "100"}}'
9496
cmds:
95-
- docker compose -f docker-compose.yml up -d
96-
- docker compose -f docker-compose.yml exec web sh -c "python manage.py populate_stts; python ./manage.py seed_db"
97+
- task: backend-up
98+
- docker compose -f docker-compose.yml exec web sh -c "python manage.py migrate && python manage.py populate_stts && python ./manage.py seed_records --clear --num {{.SEED_NUM}}"
9799
- docker compose -f docker-compose.yml down
98100

99101
backend-makemigrations:

tdrs-backend/plg/alertmanager/alertmanager.yml

Lines changed: 29 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -4,36 +4,30 @@ global:
44
smtp_from: 'no-reply@tanfdata.acf.hhs.gov'
55
smtp_auth_username: 'apikey'
66
smtp_auth_password: '{{ sendgrid_api_key }}'
7-
7+
slack_api_url: '{{ mattermost_web_hook_url }}'
88
# The directory from which notification templates are read.
99
templates:
1010
- '/etc/alertmanager/template/*.tmpl'
11-
1211
# The root route on which each incoming alert enters.
1312
route:
1413
# The labels by which incoming alerts are grouped together. For example,
1514
# multiple alerts coming in for cluster=A and alertname=LatencyHigh would
1615
# be batched into a single group.
1716
group_by: ['alertname', 'env', 'service']
18-
1917
# When a new group of alerts is created by an incoming alert, wait at
2018
# least 'group_wait' to send the initial notification.
2119
# This way ensures that you get multiple alerts for the same group that start
2220
# firing shortly after another are batched together on the first
2321
# notification.
2422
group_wait: 30s
25-
2623
# When the first notification was sent, wait 'group_interval' to send a batch
2724
# of new alerts that started firing for that group.
2825
group_interval: 5m
29-
3026
# If an alert has successfully been sent, wait 'repeat_interval' to
3127
# resend them.
3228
repeat_interval: 5m
33-
3429
# A default receiver
35-
receiver: admin-team-emails
36-
30+
receiver: mattermost
3731
# All the above attributes are inherited by all child routes and can
3832
# overwritten on each.
3933

@@ -44,18 +38,20 @@ route:
4438
- alertname=~"UpTime"
4539
receiver: dev-team-emails
4640
repeat_interval: 24h
47-
48-
# Send all severity CRITICAL/ERROR alerts to OFA admins
41+
# Send all severity CRITICAL/ERROR alerts to OFA admin emails
4942
- matchers:
50-
- severity=~"ERROR|CRITICAL"
43+
- severity=~"ERROR|CRITICAL"
5144
receiver: admin-team-emails
5245
continue: true
53-
# Send all severity CRITICAL/ERROR/WARNING alerts to TDP Devs
46+
# Send all severity CRITICAL/ERROR/WARNING alerts to mattermost and dev team emails
47+
- matchers:
48+
- severity=~"ERROR|CRITICAL|WARNING"
49+
receiver: mattermost
50+
continue: true
5451
- matchers:
55-
- severity=~"ERROR|CRITICAL|WARNING"
52+
- severity=~"ERROR|CRITICAL|WARNING"
5653
receiver: dev-team-emails
5754
continue: true
58-
5955
# Inhibition rules allow to mute a set of alerts given that another alert is
6056
# firing.
6157
# We use this to mute any warning-level notifications if the same alert is
@@ -69,22 +65,34 @@ inhibit_rules:
6965
# from both the source and target alerts,
7066
# the inhibition rule will apply!
7167
equal: [alertname, env, service]
72-
73-
- source_matchers: [alertname=~"LocalBackendDown|DevEnvironmentBackendDown|
74-
StagingBackendDown|ProductionBackendDown"]
68+
- source_matchers: [alertname=~"LocalBackendDown|DevEnvironmentBackendDown| StagingBackendDown|ProductionBackendDown"]
7569
target_match:
7670
alertname: 'UpTime'
7771
equal: ['environment', 'service']
78-
79-
80-
8172
receivers:
8273
- name: 'admin-team-emails'
8374
email_configs:
8475
- to: '{{ admin_team_emails }}'
8576
send_resolved: true
86-
8777
- name: 'dev-team-emails'
8878
email_configs:
8979
- to: '{{ dev_team_emails }}'
9080
send_resolved: true
81+
- name: 'mattermost'
82+
slack_configs:
83+
- channel: 'guest-ofa-tdp-alerts'
84+
username: 'alertmanager'
85+
send_resolved: true
86+
text: |-
87+
{{ range .Alerts -}}
88+
{{ if or (eq .Labels.severity "CRITICAL") (eq .Labels.severity "ERROR") }}
89+
@here
90+
{{ end }}
91+
*Alert:* {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }}
92+
93+
*Description:* {{ .Annotations.description }}
94+
95+
*Details:*
96+
{{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}`
97+
{{ end }}
98+
{{ end }}

tdrs-backend/plg/deploy.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ deploy_alertmanager() {
102102
cp alertmanager.yml $CONFIG
103103
SENDGRID_API_KEY=$(cf env tdp-backend-prod | grep SENDGRID | cut -d " " -f2-)
104104
yq eval -i ".global.smtp_auth_password = \"$SENDGRID_API_KEY\"" $CONFIG
105+
yq eval -i ".global.slack_api_url = \"$MATTERMOST_WEBHOOK_URL\"" $CONFIG
105106
yq eval -i ".receivers[0].email_configs[0].to = \"${ADMIN_EMAILS}\"" $CONFIG
106107
yq eval -i ".receivers[1].email_configs[0].to = \"${DEV_EMAILS}\"" $CONFIG
107108
cf push --no-route -f manifest.yml -t 180 --strategy rolling

tdrs-backend/plg/grafana_views/generate_views.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ def handle_field(field, formatted_fields, is_admin):
9999
f''' --
100100
-- Calculate if SSN is valid
101101
CASE
102-
WHEN "{field}" !~ '{regex_str}' THEN 1
102+
WHEN "{field}" IS NOT NULL AND "{field}" !~ '{regex_str}' AND "{field}" LIKE '%[^0-9]%' THEN 1
103103
ELSE 0
104104
END AS "SSN_VALID"'''.strip()
105105
)

tdrs-backend/tdpservice/data_files/serializers.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ class DataFileSerializer(serializers.ModelSerializer):
4444
has_error = serializers.SerializerMethodField()
4545
summary = DataFileSummarySerializer(many=False, read_only=True)
4646
latest_reparse_file_meta = serializers.SerializerMethodField()
47+
program_type = serializers.CharField(read_only=True)
4748

4849
class Meta:
4950
"""Metadata."""
@@ -70,9 +71,10 @@ class Meta:
7071
"summary",
7172
"latest_reparse_file_meta",
7273
"is_program_audit",
74+
"program_type",
7375
]
7476

75-
read_only_fields = ("version",)
77+
read_only_fields = ("version", "program_type")
7678

7779
def get_has_error(self, obj):
7880
"""Return whether the file has an error."""

tdrs-backend/tdpservice/data_files/test/test_api.py

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -711,3 +711,125 @@ def test_list_ofa_admin_data_file_years_no_self_stt(
711711
assert response.status_code == status.HTTP_200_OK
712712

713713
assert response.data == [2020, 2021, 2022]
714+
715+
716+
class TestDataFileQuerysetFiltering:
717+
"""Tests for the get_queryset filtering logic with program_type and is_program_audit combinations.
718+
719+
Note: Program integrity audit (is_program_audit=True) only applies to TANF files.
720+
Tribal, SSP, and FRA files do not have audit variants.
721+
"""
722+
723+
root_url = "/v1/data_files/"
724+
725+
@pytest.fixture
726+
def tanf_non_audit_file(self, data_analyst, stt):
727+
"""Create a TANF file with is_program_audit=False."""
728+
return DataFile.create_new_version(
729+
{
730+
"original_filename": "tanf_non_audit.txt",
731+
"user": data_analyst,
732+
"stt": stt,
733+
"year": 2024,
734+
"quarter": "Q1",
735+
"section": "Active Case Data",
736+
"program_type": DataFile.ProgramType.TANF,
737+
"is_program_audit": False,
738+
}
739+
)
740+
741+
@pytest.fixture
742+
def tanf_audit_file(self, data_analyst, stt):
743+
"""Create a TANF file with is_program_audit=True."""
744+
return DataFile.create_new_version(
745+
{
746+
"original_filename": "tanf_audit.txt",
747+
"user": data_analyst,
748+
"stt": stt,
749+
"year": 2024,
750+
"quarter": "Q1",
751+
"section": "Active Case Data",
752+
"program_type": DataFile.ProgramType.TANF,
753+
"is_program_audit": True,
754+
}
755+
)
756+
757+
@pytest.fixture
758+
def tribal_file(self, data_analyst, stt):
759+
"""Create a TRIBAL file (is_program_audit is always False for Tribal)."""
760+
return DataFile.create_new_version(
761+
{
762+
"original_filename": "tribal.txt",
763+
"user": data_analyst,
764+
"stt": stt,
765+
"year": 2024,
766+
"quarter": "Q1",
767+
"section": "Active Case Data",
768+
"program_type": DataFile.ProgramType.TRIBAL,
769+
"is_program_audit": False,
770+
}
771+
)
772+
773+
@pytest.mark.django_db
774+
def test_tanf_non_audit_appears_in_regular_list(
775+
self, api_client, data_analyst, stt, tanf_non_audit_file, tanf_audit_file
776+
):
777+
"""Test TANF file with is_program_audit=False appears in regular file list."""
778+
api_client.login(username=data_analyst.username, password="test_password")
779+
780+
response = api_client.get(f"{self.root_url}?stt={stt.id}&year=2024&quarter=Q1")
781+
782+
assert response.status_code == status.HTTP_200_OK
783+
file_ids = [f["id"] for f in response.data]
784+
assert len(file_ids) == 1
785+
assert tanf_non_audit_file.id in file_ids
786+
assert tanf_audit_file.id not in file_ids
787+
788+
@pytest.mark.django_db
789+
def test_tanf_audit_appears_in_program_integrity_audit_list(
790+
self, api_client, data_analyst, stt, tanf_non_audit_file, tanf_audit_file
791+
):
792+
"""Test TANF file with is_program_audit=True appears in program-integrity-audit list."""
793+
api_client.login(username=data_analyst.username, password="test_password")
794+
795+
response = api_client.get(
796+
f"{self.root_url}?stt={stt.id}&year=2024&quarter=Q1&file_type=program-integrity-audit"
797+
)
798+
799+
assert response.status_code == status.HTTP_200_OK
800+
file_ids = [f["id"] for f in response.data]
801+
assert len(file_ids) == 1
802+
assert tanf_audit_file.id in file_ids
803+
assert tanf_non_audit_file.id not in file_ids
804+
805+
@pytest.mark.django_db
806+
def test_tribal_appears_in_regular_list(
807+
self, api_client, data_analyst, stt, tribal_file, tanf_audit_file
808+
):
809+
"""Test TRIBAL file appears in regular file list alongside TANF non-audit files."""
810+
api_client.login(username=data_analyst.username, password="test_password")
811+
812+
response = api_client.get(f"{self.root_url}?stt={stt.id}&year=2024&quarter=Q1")
813+
814+
assert response.status_code == status.HTTP_200_OK
815+
file_ids = [f["id"] for f in response.data]
816+
assert len(file_ids) == 1
817+
assert tribal_file.id in file_ids
818+
assert tanf_audit_file.id not in file_ids
819+
820+
@pytest.mark.django_db
821+
def test_tribal_excluded_from_program_integrity_audit_list(
822+
self, api_client, data_analyst, stt, tribal_file, tanf_audit_file
823+
):
824+
"""Test TRIBAL files are excluded from program-integrity-audit list (only TANF audit files appear)."""
825+
api_client.login(username=data_analyst.username, password="test_password")
826+
827+
response = api_client.get(
828+
f"{self.root_url}?stt={stt.id}&year=2024&quarter=Q1&file_type=program-integrity-audit"
829+
)
830+
831+
assert response.status_code == status.HTTP_200_OK
832+
file_ids = [f["id"] for f in response.data]
833+
assert len(file_ids) == 1
834+
assert tanf_audit_file.id in file_ids
835+
assert tribal_file.id not in file_ids

tdrs-backend/tdpservice/data_files/views.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
from wsgiref.util import FileWrapper
55

66
from django.conf import settings
7-
from django.db.models import Q
87
from django.http import FileResponse, Http404, HttpResponse
98

109
from django_filters import rest_framework as filters
@@ -138,10 +137,13 @@ def get_queryset(self):
138137
)
139138
else:
140139
is_program_audit = file_type == DataFileViewSet.PIA_FILE_TYPE
141-
query = Q(program_type=DataFile.ProgramType.TANF) | Q(
142-
program_type=DataFile.ProgramType.TRIBAL
143-
) & Q(is_program_audit=is_program_audit)
144-
queryset = queryset.filter(query)
140+
queryset = queryset.filter(
141+
program_type__in=[
142+
DataFile.ProgramType.TANF,
143+
DataFile.ProgramType.TRIBAL,
144+
],
145+
is_program_audit=is_program_audit,
146+
)
145147

146148
return queryset
147149

tdrs-backend/tdpservice/fixtures/cypress/users.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -561,4 +561,4 @@
561561
"user_permissions": []
562562
}
563563
}
564-
]
564+
]

tdrs-backend/tdpservice/parsers/parser_classes/tdr_parser.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -390,9 +390,7 @@ def _generate_funded_ssn_errors(self, t2_schema, t2_records, t1_schema):
390390
for serial_num in INVALID_SSN_SERIAL_NUMBERS
391391
],
392392
error_message=(
393-
"Social Security Number is not valid. Check that the SSN is 9 digits, "
394-
"does not contain only zeroes in any one section, and does not contain "
395-
"dashes or other punctuation."
393+
"Federally funded recipients must have a valid Social Security number."
396394
),
397395
)
398396

tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -451,8 +451,7 @@
451451
validators=[
452452
category3.orValidators(
453453
[
454-
category3.isBetween(1, 4, inclusive=True),
455-
category3.isBetween(6, 9, inclusive=True),
454+
category3.isBetween(1, 9, inclusive=True),
456455
category3.isBetween(11, 12, inclusive=True),
457456
]
458457
)

0 commit comments

Comments
 (0)