updated deps, restored migrations

Author: fed1337

generate_tokens.py

@@ -1,7 +1,7 @@
 """
 Helper script to generate random values for Lemur configuration
 """
-from base64 import urlsafe_b64encode
+from base64 import urlsafe_b64encode, b64encode
 from os import urandom
 from secrets import choice, token_hex
 from string import ascii_lowercase, ascii_uppercase, digits
@@ -11,3 +11,5 @@
 print("LEMUR_ENCRYPTION_KEY:", urlsafe_b64encode(urandom(32)).decode())
 print("LEMUR_TOKEN_SECRET:", ''.join(choice(chars) for x in range(24)))
 print("SECRET:", token_hex())
+print("OAUTH2_SECRET:", token_hex())
+print("OAUTH_STATE_TOKEN_SECRET:", b64encode(urandom(32)))

lemur.conf.py

@@ -31,6 +31,12 @@
 LEMUR_TOKEN_SECRETS = [LEMUR_TOKEN_SECRET]
 LEMUR_ENCRYPTION_KEYS = ['Q7AzDsZHJRaKdS4Obeb4bLw6tYRdTqQD24xHQqJbA4A=']
 
+
+OAUTH2_SECRET = 'd105a7b3f365423a08917fa0455b353fce966e955c3a6e88f8ff149fac301a03'
+
+# this is the secret used to generate oauth state tokens
+OAUTH_STATE_TOKEN_SECRET = b'jhyNmgizEixQRnWL8F9yTfGlKz3pp2ks2GGxAUoFYE8='
+
 # REQUIRED
 # Certificate Defaults
 LEMUR_DEFAULT_COUNTRY = ""
@@ -40,6 +46,25 @@
 LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = ""
 LEMUR_SECURITY_TEAM_EMAIL = ["admin@localhost"]
 
+DIGICERT_CIS_API_KEY=""
+DIGICERT_API_KEY=""
+ENTRUST_API_USER=""
+GOOGLE_APPLICATION_CREDENTIALS=""
+DIGICERT_CIS_URL=""
+DIGICERT_URL=""
+ENTRUST_API_PASS=""
+DIGICERT_CIS_ROOTS=""
+DIGICERT_ORG_ID=""
+ENTRUST_URL=""
+DIGICERT_CIS_PROFILE_NAMES=""
+DIGICERT_ORDER_TYPE=""
+ENTRUST_ROOT=""
+ENTRUST_NAME=""
+DIGICERT_ROOT=""
+ENTRUST_EMAIL=""
+ENTRUST_PHONE=""
+
+
 # Database settings
 SQLALCHEMY_DATABASE_URI = environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://lemur:lemur@localhost:5432/lemur')
 # SQLALCHEMY_ENABLE_FLASK_REPLICATED = False
@@ -77,22 +102,19 @@
 # VERISIGN_LAST_NAME = ""
 # VERSIGN_EMAIL = ""
 
-IDP_GROUPS_KEYS = ["googleGroups"]  # a list of keys used by IDP(s) to return user groups (profile[IDP_GROUPS_KEY])
-# Note that prefix/suffix can be commented out or set to "" if no filtering against naming convention is desired
-# IDP_ROLES_PREFIX = "PREFIX-"  # prefix for all IDP-defined roles, used to match naming conventions
-# IDP_ROLES_SUFFIX = "_SUFFIX"  # suffix for all IDP-defined roles, used to match naming conventions
-# IDP_ROLES_DESCRIPTION = "Automatically generated role"  # Description to attach to automatically generated roles
-# IDP_ROLES_MAPPING = {}  # Dictionary that matches the IDP group name to the Lemur role. The Lemur role must exist.
-# Example: IDP_ROLES_MAPPING = {"security": "admin", "engineering": "operator", "jane_from_accounting": "read-only"}
-IDP_ASSIGN_ROLES_FROM_USER_GROUPS = True  # Assigns a Lemur role for each group found attached to the user
-IDP_CREATE_ROLES_FROM_USER_GROUPS = True  # Creates a Lemur role for each group found attached to the user if missing
-# Protects the built-in groups and prevents dynamically assigning users to them. Prevents IDP admin from becoming
-# Lemur admin. Use IDP_ROLES_MAPPING to create a mapping to assign these groups if desired. eg {"admin": "admin"}
-IDP_PROTECT_BUILTINS = True
-IDP_CREATE_PER_USER_ROLE = True  # Generates Lemur role for each user (allows cert assignment to a single user)
-
-# # this is the secret used to generate oauth state tokens
-# OAUTH_STATE_TOKEN_SECRET = repr(environ.get('OAUTH_STATE_TOKEN_SECRET', '')
+# IDP_GROUPS_KEYS = ["googleGroups"]  # a list of keys used by IDP(s) to return user groups (profile[IDP_GROUPS_KEY])
+# # Note that prefix/suffix can be commented out or set to "" if no filtering against naming convention is desired
+# # IDP_ROLES_PREFIX = "PREFIX-"  # prefix for all IDP-defined roles, used to match naming conventions
+# # IDP_ROLES_SUFFIX = "_SUFFIX"  # suffix for all IDP-defined roles, used to match naming conventions
+# # IDP_ROLES_DESCRIPTION = "Automatically generated role"  # Description to attach to automatically generated roles
+# # IDP_ROLES_MAPPING = {}  # Dictionary that matches the IDP group name to the Lemur role. The Lemur role must exist.
+# # Example: IDP_ROLES_MAPPING = {"security": "admin", "engineering": "operator", "jane_from_accounting": "read-only"}
+# IDP_ASSIGN_ROLES_FROM_USER_GROUPS = True  # Assigns a Lemur role for each group found attached to the user
+# IDP_CREATE_ROLES_FROM_USER_GROUPS = True  # Creates a Lemur role for each group found attached to the user if missing
+# # Protects the built-in groups and prevents dynamically assigning users to them. Prevents IDP admin from becoming
+# # Lemur admin. Use IDP_ROLES_MAPPING to create a mapping to assign these groups if desired. eg {"admin": "admin"}
+# IDP_PROTECT_BUILTINS = True
+# IDP_CREATE_PER_USER_ROLE = True  # Generates Lemur role for each user (allows cert assignment to a single user)
 
 # REDIS_HOST = 'redis'
 # REDIS_PORT = 6379

lemur/migrations/README

@@ -0,0 +1 @@
+Generic single-database configuration.

lemur/migrations/alembic.ini

@@ -0,0 +1,45 @@
+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

lemur/migrations/env.py

@@ -0,0 +1,84 @@
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+
+import alembic_autogenerate_enums
+
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+from flask import current_app
+
+db_url_escaped = current_app.config.get('SQLALCHEMY_DATABASE_URI', 'postgresql://lemur:lemur@localhost:5432/lemur').replace('%', '%%')
+config.set_main_option(
+    "sqlalchemy.url", db_url_escaped
+)
+target_metadata = current_app.extensions["migrate"].db.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(url=url)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    engine = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    connection = engine.connect()
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata,
+        **current_app.extensions["migrate"].configure_args
+    )
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.close()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

lemur/migrations/script.py.mako

@@ -0,0 +1,22 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

lemur/migrations/versions/131ec6accff5_.py

@@ -0,0 +1,39 @@
+"""Ensuring we have endpoint updated times and certificate rotation availability.
+
+Revision ID: 131ec6accff5
+Revises: e3691fc396e9
+Create Date: 2016-12-07 17:29:42.049986
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "131ec6accff5"
+down_revision = "e3691fc396e9"
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "certificates",
+        sa.Column("rotation", sa.Boolean(), nullable=False, server_default=sa.false()),
+    )
+    op.add_column(
+        "endpoints",
+        sa.Column(
+            "last_updated",
+            sa.DateTime(),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("endpoints", "last_updated")
+    op.drop_column("certificates", "rotation")
+    # ### end Alembic commands ###

lemur/migrations/versions/189e5fda5bf8_.py

@@ -0,0 +1,32 @@
+"""Adds the endpoint_dnsalias table to store CloudFront distribution aliases.
+
+Revision ID: 189e5fda5bf8
+Revises: 2201c548a5a1
+Create Date: 2021-10-25 19:52:44.133743
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '189e5fda5bf8'
+down_revision = '2201c548a5a1'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('endpoint_dnsalias',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('endpoint_id', sa.Integer(), nullable=True),
+    sa.Column('alias', sa.String(length=256), nullable=True),
+    sa.ForeignKeyConstraint(['endpoint_id'], ['endpoints.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('endpoint_dnsalias')
+    # ### end Alembic commands ###

lemur/migrations/versions/1ae8e3104db8_.py

@@ -0,0 +1,25 @@
+"""Adds additional ENUM for creating and updating certificates.
+
+Revision ID: 1ae8e3104db8
+Revises: a02a678ddc25
+Create Date: 2017-07-13 12:32:09.162800
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "1ae8e3104db8"
+down_revision = "a02a678ddc25"
+
+from alembic import op
+
+
+def upgrade():
+    op.sync_enum_values(
+        "public", "log_type", ["key_view"], ["create_cert", "key_view", "update_cert"]
+    )
+
+
+def downgrade():
+    op.sync_enum_values(
+        "public", "log_type", ["create_cert", "key_view", "update_cert"], ["key_view"]
+    )

lemur/migrations/versions/1db4f82bc780_.py

@@ -0,0 +1,45 @@
+"""Add default rotation_policy to certs where it's missing
+
+Revision ID: 1db4f82bc780
+Revises: 3adfdd6598df
+Create Date: 2018-08-03 12:56:44.565230
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "1db4f82bc780"
+down_revision = "3adfdd6598df"
+
+from alembic import op
+
+from flask import current_app
+from logging import Formatter, FileHandler, getLogger
+
+log = getLogger(__name__)
+handler = FileHandler(current_app.config.get("LOG_UPGRADE_FILE", "db_upgrade.log"))
+handler.setFormatter(
+    Formatter(
+        "%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]"
+    )
+)
+handler.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.setLevel(current_app.config.get("LOG_LEVEL", "DEBUG"))
+log.addHandler(handler)
+
+
+def upgrade():
+    connection = op.get_bind()
+
+    result = connection.execute(
+        """\
+       UPDATE certificates
+           SET rotation_policy_id=(SELECT id FROM rotation_policies WHERE name='default')
+         WHERE rotation_policy_id IS NULL
+        RETURNING id
+    """
+    )
+    log.info("Filled rotation_policy for %d certificates" % result.rowcount)
+
+
+def downgrade():
+    pass