diff --git a/apps/fhir/bluebutton/management/commands/migrate_fhir_id_to_v2.py b/apps/fhir/bluebutton/management/commands/migrate_fhir_id_to_v2.py index 007f9eac2..4742315d5 100644 --- a/apps/fhir/bluebutton/management/commands/migrate_fhir_id_to_v2.py +++ b/apps/fhir/bluebutton/management/commands/migrate_fhir_id_to_v2.py @@ -18,13 +18,13 @@ def migrate_fhir_id_to_v2(apps=None, schema_editor=None) -> None: apps_mod = apps if apps is not None else global_apps Crosswalk = apps_mod.get_model('bluebutton', 'Crosswalk') - for crosswalk in Crosswalk.objects.all(): + for crosswalk in Crosswalk.objects.iterator(chunk_size=1000): if getattr(crosswalk, '_fhir_id', None): crosswalk.fhir_id_v2 = getattr(crosswalk, '_fhir_id') # type: ignore[attr-defined] crosswalk.save() ArchivedCrosswalk = apps_mod.get_model('bluebutton', 'ArchivedCrosswalk') - for archived in ArchivedCrosswalk.objects.all(): + for archived in ArchivedCrosswalk.objects.iterator(chunk_size=1000): if getattr(archived, '_fhir_id', None): archived.fhir_id_v2 = getattr(archived, '_fhir_id') # type: ignore[attr-defined] archived.save() @@ -35,13 +35,13 @@ def reverse_migrate_v2_to_fhir_id(apps=None, schema_editor=None) -> None: apps_mod = apps if apps is not None else global_apps Crosswalk = apps_mod.get_model('bluebutton', 'Crosswalk') - for crosswalk in Crosswalk.objects.all(): + for crosswalk in Crosswalk.objects.iterator(chunk_size=1000): if getattr(crosswalk, 'fhir_id_v2', None): crosswalk._fhir_id = getattr(crosswalk, 'fhir_id_v2') # type: ignore[attr-defined] crosswalk.save() ArchivedCrosswalk = apps_mod.get_model('bluebutton', 'ArchivedCrosswalk') - for archived in ArchivedCrosswalk.objects.all(): + for archived in ArchivedCrosswalk.objects.iterator(chunk_size=1000): if getattr(archived, 'fhir_id_v2', None): archived._fhir_id = getattr(archived, 'fhir_id_v2') # type: ignore[attr-defined] archived.save() diff --git a/apps/fhir/bluebutton/migrations/0009_migrate_fhir_id_to_v2.py b/apps/fhir/bluebutton/migrations/0009_migrate_fhir_id_to_v2.py index 10a98216e..9032c3a20 100644 --- a/apps/fhir/bluebutton/migrations/0009_migrate_fhir_id_to_v2.py +++ b/apps/fhir/bluebutton/migrations/0009_migrate_fhir_id_to_v2.py @@ -1,10 +1,6 @@ # Generated by Django 4.2.24 on 2025-09-23 13:25 from django.db import migrations, models -from apps.fhir.bluebutton.management.commands.migrate_fhir_id_to_v2 import ( - migrate_fhir_id_to_v2, - reverse_migrate_v2_to_fhir_id, -) # TODO - Migration is Unapplied # This migration needs to be run after deployment of bluebutton 0008 and when all @@ -18,8 +14,7 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(migrate_fhir_id_to_v2, reverse_migrate_v2_to_fhir_id), - migrations.AddConstraint( # After migration happens, add constraint that a fhir_id must be present + migrations.AddConstraint( # After migrate_fhir_id_to_v2 is run, add constraint that a fhir_id_v2 or v3 must be present model_name='crosswalk', constraint=models.CheckConstraint( check=models.Q(fhir_id_v2__isnull=False) | models.Q(fhir_id_v3__isnull=False), diff --git a/apps/testclient/management/commands/create_test_users_and_applications_batch.py b/apps/testclient/management/commands/create_test_users_and_applications_batch.py index 26bb590f9..880abfa48 100755 --- a/apps/testclient/management/commands/create_test_users_and_applications_batch.py +++ b/apps/testclient/management/commands/create_test_users_and_applications_batch.py @@ -5,7 +5,7 @@ import apps.logging.request_logger as logging -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from os import listdir from random import randint, randrange, choice, sample from django.contrib.auth.models import Group @@ -34,18 +34,20 @@ 'capability-a', 'capability-b'] APPLICATION_SCOPES_NON_DEMOGRAPHIC = ['patient/ExplanationOfBenefit.read', 'patient/Coverage.read', 'capability-a', 'capability-b'] -DEFAULT_BENE_COUNT = 100 + +# Keep up with prod count +DEFAULT_BENE_COUNT = 675000 DEFAULT_DEV_COUNT = 150 DEFAULT_MAX_APPS_PER_DEV = 5 -def create_group(name="BlueButton"): +def create_group(name='BlueButton'): g, created = Group.objects.get_or_create(name=name) if created: - print("%s group created" % (name)) + print('%s group created' % (name)) else: - print("%s group pre-existing. Create skipped." % (name)) + print('%s group pre-existing. Create skipped.' % (name)) return g # To avoid naming collisions when running this command more than once @@ -63,103 +65,127 @@ def get_first_available_number(firstname): def create_dev_users_apps_and_bene_crosswalks( - group, bene_count, dev_count, app_max, refresh_count, archived_tokens, - archived_grants + archived_grants, + auto_generate ): - # - # generate dev users dev0001, dev0002, dev0003 etc, with password, email, security questions, etc. - # Counts are based on the inputs, but if undefined,will default to the defaults defined above - # each dev user can have 1 to many applications: dev0001-app01, dev0001-app02 - # generate crosswalk bene users with FHIR-ID. HICN-HASH, MBI-HASH etc - # - - # effectively turn off logging + """Creates bene and dev users, associated apps, and crosswalks for testing + + Args: + bene_count (_type_): _description_ + dev_count (_type_): _description_ + app_max (_type_): _description_ + refresh_count (_type_): _description_ + archived_tokens (_type_): _description_ + archived_grants (_type_): _description_ + auto_generate (_type_): _description_ + """ mymedicare_cb_logger.setLevel(logging.CRITICAL) outreach_logger.setLevel(logging.CRITICAL) + group = create_group() bene_pk_list = [] - files = [f for f in listdir('./synthetic-data') if re.match(r'synthetic-beneficiary-.*\.rif', f)] + count = 0 file_cnt = 0 synthetic_bene_cnt = 0 - begin_number = get_first_available_number("bene") - - for f in files: - print("file={}".format(f)) - bene_rif = open('./synthetic-data/{}'.format(f), 'r') - while True: - if count == bene_count: - break - line = bene_rif.readline() - if not line: + begin_number = get_first_available_number('bene') + if auto_generate: + for i in range(bene_count): + if i % 10000 == 0: + print("we have made it through: ", i, " fake beneficiaries") + id = str(i).zfill(10) + fn = f'FN{id}' + ln = f'LN{id}' + fhir_id = f'-{id}' + args = { + 'username': str(uuid.uuid4()), + 'user_hicn_hash': hash_hicn(id), + 'user_mbi': f'MBI{id}', + 'user_id_type': 'H', + 'fhir_id_v2': fhir_id, + 'first_name': fn, + 'last_name': ln, + 'email': f'{fn}.{ln}@xyz.com' + } + slsx_client = OAuth2ConfigSLSx(args) + user = create_beneficiary_record(slsx_client, fhir_id) + bene_pk_list.append(user.pk) + else: + files = [f for f in listdir('./synthetic-data') if re.match(r'synthetic-beneficiary-.*\.rif', f)] + for f in files: + print('file={}'.format(f)) + bene_rif = open('./synthetic-data/{}'.format(f), 'r') + while True: + if count == bene_count: + break + line = bene_rif.readline() + if not line: + break + if line.startswith('DML_IND|BENE_ID'): + continue + else: + flds = line.split('|') + fhir_id = flds[1] + mbi = flds[14] + hicn = flds[18] + fn = f'bene{count + begin_number}' + ln = f'user{count + begin_number}' + + # skip fred + if fhir_id != '-20140000008325': + args = { + 'username': str(uuid.uuid1()), + 'user_hicn_hash': hash_hicn(hicn), + 'user_mbi': mbi, + 'user_id_type': 'H', + 'fhir_id_v2': fhir_id, + 'first_name': fn, + 'last_name': ln, + 'email': fn + '.' + ln + '@xyz.net', + } + slsx_client = OAuth2ConfigSLSx(args) + try: + u = create_beneficiary_record(slsx_client, fhir_id) + date_picked = datetime.now(timezone.utc) - timedelta(days=randrange(700)) + u.date_joined = date_picked.replace(tzinfo=pytz.utc) + u.save() + c = Crosswalk.objects.get(user=u) + c.created = u.date_joined + c.save() + bene_pk_list.append(u.pk) + synthetic_bene_cnt += 1 + count += 1 + except ValidationError: + # If there is something wrong during 'create_beneficiary_record' + # i.e. 'user already exists', just try the next .rif record + continue + bene_rif.close() + file_cnt += 1 + print('RIF file processed = {}, synthetic bene generated = {}'.format(file_cnt, synthetic_bene_cnt)) + if file_cnt >= 1: break - if line.startswith('DML_IND|BENE_ID'): - continue - else: - flds = line.split('|') - fhir_id = flds[1] - mbi = flds[14] - hicn = flds[18] - fn = "bene{}".format(count + begin_number) - ln = "user{}".format(count + begin_number) - - # skip fred - if fhir_id != '-20140000008325': - args = { - 'username': str(uuid.uuid1()), - 'user_hicn_hash': hash_hicn(hicn), - 'user_mbi': mbi, - 'user_id_type': 'H', - 'fhir_id_v2': fhir_id, - 'first_name': fn, - 'last_name': ln, - 'email': fn + '.' + ln + '@xyz.net', - } - slsx_client = OAuth2ConfigSLSx(args) - try: - u = create_beneficiary_record(slsx_client, fhir_id) - date_picked = datetime.utcnow() - timedelta(days=randrange(700)) - u.date_joined = date_picked.replace(tzinfo=pytz.utc) - u.save() - c = Crosswalk.objects.get(user=u) - c.created = u.date_joined - c.save() - bene_pk_list.append(u.pk) - synthetic_bene_cnt += 1 - count += 1 - print(".", end="", flush=True) - time.sleep(.05) - except ValidationError: - # If there is something wrong during 'create_beneficiary_record' - # i.e. 'user already exists', just try the next .rif record - continue - bene_rif.close() - file_cnt += 1 - print("RIF file processed = {}, synthetic bene generated = {}".format(file_cnt, synthetic_bene_cnt)) - if file_cnt >= 1: - break # create dev users according dev-count parameter, default to 100 # generate access tokens + refresh tokens + archived tokens for random picked benes for each app app_index = 0 - begin_number = get_first_available_number("DevUserFN") + begin_number = get_first_available_number('DevUserFN') for i in range(dev_count): - dev_u_fn = "DevUserFN{}".format(i + begin_number) - dev_u_ln = "DevUserLN{}".format(i + begin_number) - u = User.objects.create_user(username="{}.{}".format(dev_u_fn, dev_u_ln), + dev_u_fn = 'DevUserFN{}'.format(i + begin_number) + dev_u_ln = 'DevUserLN{}'.format(i + begin_number) + u = User.objects.create_user(username='{}.{}'.format(dev_u_fn, dev_u_ln), first_name=dev_u_fn, last_name=dev_u_ln, email='{}.{}@example.com'.format(dev_u_fn, dev_u_ln), - password="THEP@ssw0rd{}".format(i),) + password='THEP@ssw0rd{}'.format(i),) UserProfile.objects.create(user=u, - user_type="DEV", + user_type='DEV', create_applications=True, - organization_name=u.username + "ACME Inc.", + organization_name=u.username + 'ACME Inc.', password_reset_question_1='1', password_reset_answer_1='blue', password_reset_question_2='2', @@ -169,22 +195,21 @@ def create_dev_users_apps_and_bene_crosswalks( u.groups.add(group) # apps per DEV user app_cnt = randint(1, app_max) if app_max > 0 else 0 - print(">>>>generating apps for user={}".format(u.username)) for i in range(app_cnt): app_index += 1 - app_name = "app{}_{}".format(i, u) - redirect_uri = "{}/testclient_{}/callback".format(settings.HOSTNAME_URL, app_name) - if not (redirect_uri.startswith("http://") or redirect_uri.startswith("https://")): - redirect_uri = "https://" + redirect_uri + app_name = 'app{}_{}'.format(i, u) + redirect_uri = '{}/testclient_{}/callback'.format(settings.HOSTNAME_URL, app_name) + if not(redirect_uri.startswith('http://') or redirect_uri.startswith('https://')): + redirect_uri = 'https://' + redirect_uri # 2% inactive, 5% opt out demo scopes # 10% public/implicit 90% confidential/authorization-code - cl_type = "confidential" - auth_grant_type = "authorization-code" + cl_type = 'confidential' + auth_grant_type = 'authorization-code' if app_index % 10 == 0: - cl_type = "public" - auth_grant_type = "implicit" + cl_type = 'public' + auth_grant_type = 'implicit' a = Application.objects.create(name=app_name, redirect_uris=redirect_uri, @@ -199,16 +224,15 @@ def create_dev_users_apps_and_bene_crosswalks( u.save() a.save() titles = [ - "My Medicare and supplemental coverage information.", - "My Medicare claim information.", - "My general patient and demographic information.", - "Profile information including name and email." + 'My Medicare and supplemental coverage information.', + 'My Medicare claim information.', + 'My general patient and demographic information.', + 'Profile information including name and email.' ] for t in titles: c = ProtectedCapability.objects.get(title=t) a.scope.add(c) - print("<<<<=3.2 -setuptools>=40.8.0 +setuptools>=40.8.0 \ No newline at end of file