diff --git a/.github/workflows/containers.yaml b/.github/workflows/containers.yaml index 1fda72a06..f34598c60 100644 --- a/.github/workflows/containers.yaml +++ b/.github/workflows/containers.yaml @@ -33,6 +33,7 @@ jobs: - dnsmasq - ironic-nautobot-client - understack-tests + - nautobot uses: ./.github/workflows/build-container-reuse.yaml secrets: inherit with: diff --git a/ansible/roles/openstack_octavia/tasks/main.yml b/ansible/roles/openstack_octavia/tasks/main.yml index a35b01884..1fa60a392 100644 --- a/ansible/roles/openstack_octavia/tasks/main.yml +++ b/ansible/roles/openstack_octavia/tasks/main.yml @@ -91,7 +91,7 @@ - name: Build octavia_port_names (override or hostname) ansible.builtin.set_fact: - octavia_port_names: "{{ octavia_port_names | default([]) + [ (hostvars[item].octavia_host_override | default(item)) ] }}" + octavia_port_names: "{{ octavia_port_names | default([]) + [hostvars[item].octavia_host_override | default(item)] }}" loop: "{{ groups['all'] }}" when: - hostvars[item].ovs_enabled is defined diff --git a/containers/nautobot/Dockerfile b/containers/nautobot/Dockerfile new file mode 100644 index 000000000..7110fd664 --- /dev/null +++ b/containers/nautobot/Dockerfile @@ -0,0 +1,20 @@ +# renovate: datasource=docker depName=networktocode/nautobot versioning=semver +ARG NAUTOBOT_VERSION=3.0.2 +ARG PYTHON_VERSION=3.12 +ARG NAUTOBOT_CONTAINER_TYPE="" + +FROM ghcr.io/nautobot/nautobot${NAUTOBOT_CONTAINER_TYPE}:${NAUTOBOT_VERSION}-py${PYTHON_VERSION} AS prod + +LABEL org.opencontainers.image.source=https://github.com/rackerlabs/understack + +ENV PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 + +# install plugin_requirements to /opt/nautobot/ per the Nautobot Docker Compose example +COPY ./containers/nautobot/plugin_requirements.txt /opt/nautobot/ +RUN pip install --no-warn-script-location -r /opt/nautobot/plugin_requirements.txt +# copy in a sample config +COPY ./containers/nautobot/nautobot_config.py /opt/nautobot_config/nautobot_config.py +# copy in the healthcheck script for celery beats +COPY ./containers/nautobot/scripts/nautobot_celery_beats_hcheck.py /opt/nautobot/ diff --git a/containers/nautobot/README.md b/containers/nautobot/README.md new file mode 100644 index 000000000..9d0864739 --- /dev/null +++ b/containers/nautobot/README.md @@ -0,0 +1,6 @@ +# nautobot container builds + +``` bash +docker compose -f docker-compose.nautobot.yml -f docker-compose.postgres.yml -f docker-compose.redis.yml build +docker compose -f docker-compose.nautobot.yml -f docker-compose.postgres.yml -f docker-compose.redis.yml up +``` diff --git a/containers/nautobot/default-creds.env b/containers/nautobot/default-creds.env new file mode 100644 index 000000000..de7fbcf24 --- /dev/null +++ b/containers/nautobot/default-creds.env @@ -0,0 +1,10 @@ +# default creds file + +NAUTOBOT_DB_PASSWORD=changeme +NAUTOBOT_REDIS_PASSWORD=changeme +NAUTOBOT_SECRET_KEY=changeme + +NAUTOBOT_SUPERUSER_NAME=admin +NAUTOBOT_SUPERUSER_EMAIL=admin@example.com +NAUTOBOT_SUPERUSER_PASSWORD=admin +NAUTOBOT_SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567 diff --git a/containers/nautobot/development.env b/containers/nautobot/development.env new file mode 100644 index 000000000..abab200ba --- /dev/null +++ b/containers/nautobot/development.env @@ -0,0 +1,43 @@ +################################################################################ +# DEV File: Store environment information. NOTE: Secrets NOT stored here! +################################################################################ +# Nautobot Configuration Environment Variables +NAUTOBOT_ALLOWED_HOSTS=* +NAUTOBOT_BANNER_TOP="Local" +NAUTOBOT_CHANGELOG_RETENTION=0 +NAUTOBOT_CONFIG=/opt/nautobot_config/nautobot_config.py + +NAUTOBOT_CREATE_SUPERUSER=true +NAUTOBOT_DEBUG=True +NAUTOBOT_DJANGO_EXTENSIONS_ENABLED=True +NAUTOBOT_DJANGO_TOOLBAR_ENABLED=True +NAUTOBOT_LOG_LEVEL=DEBUG +NAUTOBOT_METRICS_ENABLED=True +NAUTOBOT_NAPALM_TIMEOUT=5 +NAUTOBOT_MAX_PAGE_SIZE=0 + +# Redis Configuration Environment Variables +NAUTOBOT_REDIS_HOST=redis +NAUTOBOT_REDIS_PORT=6379 +# Uncomment NAUTOBOT_REDIS_SSL if using SSL +# NAUTOBOT_REDIS_SSL=True + +# Nautobot DB Connection Environment Variables +NAUTOBOT_DB_ENGINE=django.db.backends.postgresql +NAUTOBOT_DB_NAME=nautobot +NAUTOBOT_DB_USER=nautobot +NAUTOBOT_DB_HOST=db +NAUTOBOT_DB_TIMEOUT=300 + +# Use them to overwrite the defaults in nautobot_config.py +# NAUTOBOT_DB_ENGINE=django.db.backends.postgresql +# NAUTOBOT_DB_PORT=5432 + +# Needed for Postgres should match the values for Nautobot above +POSTGRES_USER=${NAUTOBOT_DB_USER} +POSTGRES_DB=${NAUTOBOT_DB_NAME} + +# Needed to initialize the Postgres super user, use the same for dev +POSTGRES_PASSWORD=${NAUTOBOT_DB_PASSWORD} + +NAUTOBOT_SSOT_HIDE_EXAMPLE_JOBS="True" diff --git a/containers/nautobot/docker-compose.nautobot.yml b/containers/nautobot/docker-compose.nautobot.yml new file mode 100644 index 000000000..70508e97f --- /dev/null +++ b/containers/nautobot/docker-compose.nautobot.yml @@ -0,0 +1,73 @@ +--- +version: "3.8" +name: understack-nautobot +services: + nautobot: + image: local/understack-nautobot:latest + build: + context: ../../. + dockerfile: containers/nautobot/Dockerfile + env_file: + - "default-creds.env" + - "development.env" + environment: + - PYTHONDONTWRITEBYTECODE=1 + ports: + - "8000:8080" + depends_on: + redis: + condition: "service_started" + db: + condition: "service_healthy" + healthcheck: + interval: "30s" + timeout: "30s" + start_period: "30s" + retries: 15 + worker: + image: local/understack-nautobot:latest + build: + context: ../../. + dockerfile: containers/nautobot/Dockerfile + entrypoint: + - "sh" + - "-c" # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env and $$ because of docker-compose + - "nautobot-server celery worker -l $$NAUTOBOT_LOG_LEVEL --events" + env_file: + - "default-creds.env" + - "development.env" + environment: + - PYTHONDONTWRITEBYTECODE=1 + depends_on: + nautobot: + condition: "service_started" + healthcheck: + interval: "30s" + timeout: "30s" # its taking a long time to start sometimes + start_period: "30s" + retries: 10 + test: ["CMD", "bash", "-c", "nautobot-server celery inspect ping --destination celery@$$HOSTNAME"] ## $$ because of docker-compose + + nautobot-celery-beats: + image: local/understack-nautobot:latest + build: + context: ../../. + dockerfile: containers/nautobot/Dockerfile + environment: + - PYTHONDONTWRITEBYTECODE=1 + entrypoint: + - "sh" + - "-c" # this is to evaluate the $NAUTOBOT_LOG_LEVEL from the env and $$ because of docker-compose + - "nautobot-server celery beat -l $$NAUTOBOT_LOG_LEVEL" + env_file: + - "default-creds.env" + - "development.env" + depends_on: + nautobot: + condition: "service_healthy" + healthcheck: + interval: "30s" + timeout: "10s" + start_period: "30s" + retries: 3 + test: ["CMD", "python", "/opt/nautobot/nautobot_celery_beats_hcheck.py"] diff --git a/containers/nautobot/docker-compose.postgres.yml b/containers/nautobot/docker-compose.postgres.yml new file mode 100644 index 000000000..7c9bbffc4 --- /dev/null +++ b/containers/nautobot/docker-compose.postgres.yml @@ -0,0 +1,22 @@ +--- +version: "3.8" +name: understack-nautobot +services: + nautobot: + environment: + - "NAUTOBOT_DB_ENGINE=django.db.backends.postgresql" + db: + image: "postgres:16-alpine" + env_file: + - "default-creds.env" + - "development.env" + volumes: + - "postgres_data:/var/lib/postgresql/data" + healthcheck: + test: "pg_isready --username=$$POSTGRES_USER --dbname=$$POSTGRES_DB" + interval: "10s" + timeout: "5s" + retries: 10 + +volumes: + postgres_data: {} diff --git a/containers/nautobot/docker-compose.redis.yml b/containers/nautobot/docker-compose.redis.yml new file mode 100644 index 000000000..dcb503680 --- /dev/null +++ b/containers/nautobot/docker-compose.redis.yml @@ -0,0 +1,13 @@ +--- +version: "3.8" +name: understack-nautobot +services: + redis: + image: "redis:7-alpine" + command: + - "sh" + - "-c" # this is to evaluate the $NAUTOBOT_REDIS_PASSWORD from the env + - "redis-server --appendonly yes --requirepass $$NAUTOBOT_REDIS_PASSWORD" + env_file: + - "default-creds.env" + - "development.env" diff --git a/containers/nautobot/nautobot_config.py b/containers/nautobot/nautobot_config.py new file mode 100644 index 000000000..a882addaa --- /dev/null +++ b/containers/nautobot/nautobot_config.py @@ -0,0 +1,220 @@ +import os + +from nautobot.core.settings import * # noqa F401,F403 +from nautobot.core.settings import DATABASES + +######################### +# # +# Required settings # +# # +######################### + +# Ensure proper Unicode handling for MySQL +# +if DATABASES["default"]["ENGINE"] == "django.db.backends.mysql": + DATABASES["default"]["OPTIONS"] = {"charset": "utf8mb4"} + +# This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. +# For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and +# symbols. Nautobot will not run without this defined. For more information, see +# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY +SECRET_KEY = os.getenv( + "NAUTOBOT_SECRET_KEY", + "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ", +) + +##################################### +# # +# Optional Django core settings # +# # +##################################### + +# Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs: +# https://docs.djangoproject.com/en/stable/topics/logging/ +# +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "normal": { + "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)s :\n %(message)s", + "datefmt": "%H:%M:%S", + }, + "verbose": { + "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)-20s %(filename)-15s %(funcName)30s() :\n %(message)s", + "datefmt": "%H:%M:%S", + }, + }, + "handlers": { + "normal_console": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "normal", + }, + "verbose_console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "verbose", + }, + }, + "root": { + "handlers": ["verbose_console"], + "level": "DEBUG", + }, +} + +################################################################### +# # +# Optional settings specific to Nautobot and its related apps # +# # +################################################################### + + +# SSO via Dex +def _read_cred(filename): + try: + with open(filename) as cred: + return cred.read().strip() + except FileNotFoundError: + return None + + +SOCIAL_AUTH_OIDC_OIDC_ENDPOINT = _read_cred("/opt/nautobot/sso/issuer") or os.getenv( + "SOCIAL_AUTH_OIDC_OIDC_ENDPOINT" +) +SOCIAL_AUTH_OIDC_KEY = _read_cred("/opt/nautobot/sso/client-id") or "nautobot" +SOCIAL_AUTH_OIDC_SECRET = _read_cred("/opt/nautobot/sso/client-secret") +# The “openid”, “profile” and “email” are requested by default, +# below *adds* scope. +SOCIAL_AUTH_OIDC_SCOPE = ["groups"] + +if SOCIAL_AUTH_OIDC_OIDC_ENDPOINT and SOCIAL_AUTH_OIDC_SECRET: + AUTHENTICATION_BACKENDS = [ + "social_core.backends.open_id_connect.OpenIdConnectAuth", + "nautobot.core.authentication.ObjectPermissionBackend", + ] + +# EXTERNAL_AUTH_DEFAULT_GROUPS = [] +# EXTERNAL_AUTH_DEFAULT_PERMISSIONS = {} + +# Directory where cloned Git repositories will be stored. +# +# GIT_ROOT = os.getenv("NAUTOBOT_GIT_ROOT", os.path.join(NAUTOBOT_ROOT, "git").rstrip("/")) + +# Prefixes to use for custom fields, relationships, and computed fields in GraphQL representation of data. +# +# GRAPHQL_COMPUTED_FIELD_PREFIX = "cpf" +# GRAPHQL_CUSTOM_FIELD_PREFIX = "cf" +# GRAPHQL_RELATIONSHIP_PREFIX = "rel" + +# Set to True to hide rather than disabling UI elements that a user doesn't have permission to access. +# +# HIDE_RESTRICTED_UI = False + +# HTTP proxies Nautobot should use when sending outbound HTTP requests (e.g. for webhooks). +# +# HTTP_PROXIES = { +# 'http': 'http://10.10.1.10:3128', +# 'https': 'http://10.10.1.10:1080', +# } + +# Directory where Jobs can be discovered. +# +# JOBS_ROOT = os.getenv("NAUTOBOT_JOBS_ROOT", os.path.join(NAUTOBOT_ROOT, "jobs").rstrip("/")) + +# Log Nautobot deprecation warnings. Note that this setting is ignored (deprecation logs always enabled) if DEBUG = True +# +# LOG_DEPRECATION_WARNINGS = is_truthy(os.getenv("NAUTOBOT_LOG_DEPRECATION_WARNINGS", "False")) + +# Setting this to True will display a "maintenance mode" banner at the top of every page. +# +# MAINTENANCE_MODE = is_truthy(os.getenv("NAUTOBOT_MAINTENANCE_MODE", "False")) + +# Maximum number of objects that the UI and API will retrieve in a single request. +# +# MAX_PAGE_SIZE = 1000 + +# Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' +# +# METRICS_ENABLED = is_truthy(os.getenv("NAUTOBOT_METRICS_ENABLED", "False")) + +# Credentials that Nautobot will uses to authenticate to devices when connecting via NAPALM. +# +NAPALM_USERNAME = os.environ.get("NAUTOBOT_NAPALM_USERNAME", "admin") +NAPALM_PASSWORD = os.environ.get("NAUTOBOT_NAPALM_PASSWORD", "admin") +NAPALM_TIMEOUT = int(os.environ.get("NAUTOBOT_NAPALM_TIMEOUT", "30")) + +# NAPALM optional arguments (see https://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must +# be provided as a dictionary. +# +# NAPALM_ARGS = {} + +# Default number of objects to display per page of the UI and REST API. +# +# PAGINATE_COUNT = 50 + +# Options given in the web UI for the number of objects to display per page. +# +# PER_PAGE_DEFAULTS = [25, 50, 100, 250, 500, 1000] + +# Enable installed plugins. Add the name of each plugin to the list. +# +# PLUGINS = [] +PLUGINS = [ + "nautobot_plugin_nornir", + "nautobot_golden_config", +] + +PLUGINS_CONFIG = { + "nautobot_plugin_nornir": { + "username": "admin", + "password": "admin", + "secret": "admin", + "nornir_settings": { + "credentials": "nautobot_plugin_nornir.plugins.credentials.nautobot_secrets.CredentialsNautobotSecrets", + "username": "admin", + "runner": { + "plugin": "threaded", + "options": { + "num_workers": 20, + }, + }, + }, + "use_config_context": { + "connection_options": True, + }, + "connection_options": { + "napalm": { + "extras": { + "optional_args": { + "port": 41268, + }, + }, + }, + }, + }, + "nautobot_golden_config": { + "per_feature_bar_width": 0.15, + "per_feature_width": 13, + "per_feature_height": 4, + "enable_backup": True, + "enable_compliance": True, + "enable_intended": True, + "enable_sotagg": True, + "sot_agg_transposer": None, + "enable_postprocessing": True, + "postprocessing_callables": [], + "postprocessing_subscribed": [], + "platform_slug_map": None, + }, + "dispatcher_mapping": { + "default": "nornir_nautobot.plugins.tasks.dispatcher.default.NautobotNornirDriver", + "default_netmiko": "nornir_nautobot.plugins.tasks.dispatcher.default.NetmikoNautobotNornirDriver", + "cisco_asa": "nornir_nautobot.plugins.tasks.dispatcher.cisco_asa.NautobotNornirDriver", + "cisco_nxos": "nornir_nautobot.plugins.tasks.dispatcher.cisco_nxos.NautobotNornirDriver", + "cisco_ios": "nornir_nautobot.plugins.tasks.dispatcher.cisco_ios.NautobotNornirDriver", + "cisco_xr": "nornir_nautobot.plugins.tasks.dispatcher.cisco_xr.NautobotNornirDriver", + "juniper_junos": "nornir_nautobot.plugins.tasks.dispatcher.juniper_junos.NautobotNornirDriver", + "arista_eos": "nornir_nautobot.plugins.tasks.dispatcher.arista_eos.NautobotNornirDriver", + }, +} diff --git a/containers/nautobot/plugin_requirements.txt b/containers/nautobot/plugin_requirements.txt new file mode 100644 index 000000000..ed59e9027 --- /dev/null +++ b/containers/nautobot/plugin_requirements.txt @@ -0,0 +1,5 @@ +# Commit versions below are needed for some packages as the latest version of +# them are not running on python 3.12 yet. +nautobot[sso]==3.0.2 +nautobot-golden-config==3.0.0 +nautobot-plugin-nornir==3.0.0 diff --git a/containers/nautobot/scripts/nautobot_celery_beats_hcheck.py b/containers/nautobot/scripts/nautobot_celery_beats_hcheck.py new file mode 100644 index 000000000..b827cd1f7 --- /dev/null +++ b/containers/nautobot/scripts/nautobot_celery_beats_hcheck.py @@ -0,0 +1,19 @@ +import os +from datetime import datetime +from datetime import timedelta +from zoneinfo import ZoneInfo + +from celery import current_app +from celery.beat import Service + +os.chdir("/opt/nautobot") + +schedule = Service(current_app).get_scheduler().get_schedule() + +now = datetime.now(tz=ZoneInfo("UTC")) +for task_name, task in schedule.items(): + # Check if any of the tasks are overdue + try: + assert now < task.last_run_at + task.schedule.run_every + except AttributeError: + assert timedelta() < task.schedule.remaining_estimate(task.last_run_at)