From a22fe3b2cabaac4b7da3507561c6840aa87fcced Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 26 Feb 2026 22:41:59 -0600 Subject: [PATCH 1/8] fix(naming): use the updated makefile target name --- .pre-commit-config.yaml | 11 +- Makefile | 10 +- compose.yml | 1 - config/settings/base.py | 1 + docs/environment_variables.md | 132 +++++----- pyproject.toml | 1 + scripts/extract_env_vars.py | 341 +++++++++++++++++++++++++ scripts/tests/test_extract_env_vars.py | 104 ++++++++ 8 files changed, 524 insertions(+), 77 deletions(-) create mode 100755 scripts/extract_env_vars.py create mode 100644 scripts/tests/test_extract_env_vars.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7419f87..cc35ddcf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ exclude: 'docs|migrations|.git|.tox' -default_stages: [commit] +default_stages: [pre-commit] fail_fast: true repos: @@ -15,3 +15,12 @@ repos: - id: ruff-format - id: ruff-check args: [--fix, --exit-non-zero-on-fix] + + - repo: local + hooks: + - id: update-env-docs + name: Check Environment Variables Markdown + entry: make update-env-docs CHECK=true + language: system + pass_filenames: false + always_run: true diff --git a/Makefile b/Makefile index c5b867f6..560c7f6a 100644 --- a/Makefile +++ b/Makefile @@ -175,4 +175,12 @@ copy-libs: @docker compose cp translator:/app/attribute_pb2_grpc.py translator/ @docker compose cp translator:/app/capability_pb2.py translator/ @docker compose cp translator:/app/capability_pb2.pyi translator/ - @docker compose cp translator:/app/capability_pb2_grpc.py translator/ + +## update-env-docs: update environment variable documentation append CHECK=true to get a diff if not up to date +.Phony: update-env-docs +update-env-docs: +ifeq ($(CHECK),true) + @uv run scripts/extract_env_vars.py --check +else + @uv run scripts/extract_env_vars.py +endif diff --git a/compose.yml b/compose.yml index 5aa91710..f08f4150 100644 --- a/compose.yml +++ b/compose.yml @@ -1,5 +1,4 @@ --- - services: django: build: diff --git a/config/settings/base.py b/config/settings/base.py index 289f5f59..8bfc90e8 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -290,6 +290,7 @@ CORS_URLS_REGEX = r"^/api/.*$" # Your stuff... # ------------------------------------------------------------------------------ + # Are you using local passwords or oidc? AUTH_METHOD = os.environ.get("SCRAM_AUTH_METHOD", "local").lower() diff --git a/docs/environment_variables.md b/docs/environment_variables.md index c44629a7..e99bde4e 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -1,76 +1,60 @@ -## Environment Variables to Set for Deployment -[comment]: # Which branch of SCRAM to use (you probably want to set it to a release tag) -scram_code_branch: -#### Systems -[comment]: # Email of the main admin -scram_manager_email: -[comment]: # Set to true for production mode; set to false to set up the compose.override.local.yml stack -scram_prod: true -[comment]: # Set to true if you want ansible to install a scram user -scram_install_user: true -[comment]: # What group to put `scram` user in -scram_group: 'scram' -[comment]: # What username to use for `scram` user -scram_user: '' -[comment]: # WHat uid to use for `scram` user -scram_uid: '' -[comment]: # What directory to use for base of the repo -scram_home: '/usr/local/scram' -[comment]: # IP or DNS record for your postgres host -scram_postgres_host: -[comment]: # What postgres user to use -scram_postgres_user: '' +# Environment Variables Reference -#### Authentication -[comment]: # This chooses if you want to use oidc or local accounts. This can be local or oidc only. Default: `local` -scram_auth_method: "local" -[comment]: # This client id (username) for your oidc connection. Only need to set this if you are trying to do oidc. -scram_oidc_client_id: +To update, run `make update-env-docs`. -#### Networking -[comment]: # What is the peering interface docker uses for gobgp to talk to the router -scram_peering_iface: 'ens192' -[comment]: # The v6 network of your peering connection -scram_v4_subnet: '10.0.0.0/24' -[comment]: # The v4 IP of the peering connection for the router side -scram_v4_gateway: '10.0.0.1' -[comment]: # The v4 IP of the peering connection for gobgp side -scram_v4_address: '10.0.0.2' -[comment]: # The v6 network of your peering connection -scram_v6_subnet: '2001:db8::/64' -[comment]: # The v6 IP of the peering connection for the router side -scram_v6_gateway: '2001:db8::2' -[comment]: # The v6 IP of the peering connection for the gobgp side -scram_v6_address: '2001:db8::3' -[comment]: # The AS you want to use for gobgp -scram_as: -[comment]: # A string representing your gobgp instance. Often seen as the local IP of the gobgp instance -scram_router_id: -[comment]: # -scram_peer_as: -[comment]: # The AS you want to use for gobgp side (can this be the same as `scram_as`?) -scram_local_as: -[comment]: # The fqdn of the server hosting this - to be used for nginx -scram_nginx_host: -[comment]: # List of allowed hosts per the django setting "ALLOWED_HOSTS". This should be a list of strings in shell -[comment]: # `django` is required for the websockets to work -[comment]: # Our Ansible assumes `django` + `scram_nginx_host` -scram_django_allowed_hosts: "django" -[comment]: # The fqdn of the server hosting this - to be used for nginx -scram_server_alias: -[comment]: # Do you want to set an md5 for authentication of bgp -scram_bgp_md5_enabled: false -[comment]: # The neighbor config of your gobgp config -scram_neighbors: -[comment]: # The v6 address of your neighbor - - neighbor_address: 2001:db8::2 -[comment]: # This is a v6 address so don't use v4 - ipv4: false -[comment]: # This is a v6 address so use v6 - ipv6: true -[comment]: # The v4 address of your neighbor - - neighbor_address: 10.0.0.200 -[comment]: # This is a v4 address so use v4 - ipv4: true -[comment]: # This is a v4 address so don't use v6 - ipv6: false +| Variable | Service | Environments | Default | file | Description | +| --- | --- | --- | --- | --- | --- | +| `CELERY_BEAT_REPLICAS` | Compose | Common | 0 | [compose.yml](file://compose.yml) | - | +| `CELERY_WORKER_REPLICAS` | Compose | Common | 0 | [compose.yml](file://compose.yml) | - | +| `DEBUG` | Compose | Multiple | - | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.yml](file://compose.override.yml) | Here we setup a debugger if this is desired. This obviously should not be run in production | +| `DJANGO_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | +| `DOCS_PORT` | Compose | Multiple | 8888 | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.yml](file://compose.override.yml) | - | +| `FLOWER_REPLICAS` | Compose | Common | 0 | [compose.yml](file://compose.yml) | - | +| `GOBGP_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | +| `HOSTNAME` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `POSTGRES_ENABLED` | Compose | Common | 1 | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.production.yml](file://compose.override.production.yml), [compose.override.yml](file://compose.override.yml), [compose.yml](file://compose.yml) | - | +| `REDIS_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | +| `SCRAM_PEERING_IFACE` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `SCRAM_V4_ADDRESS` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `SCRAM_V4_GATEWAY` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `SCRAM_V4_SUBNET` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `SCRAM_V6_ADDRESS` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `SCRAM_V6_GATEWAY` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `SCRAM_V6_SUBNET` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | +| `TRANSLATOR_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | +| `CONN_MAX_AGE` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | noqa F405 | +| `DATABASE_URL` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py), [config/settings/production.py](file://config/settings/production.py) | DATABASES https docs.djangoproject.com/en/dev/ref/settings databases | +| `DEBUG` | Django | Unknown | - | [config/asgi.py](file://config/asgi.py) | Here we setup a debugger if this is desired. This obviously should not be run in production | +| `DJANGO_ADMIN_URL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | ADMIN Django Admin URL regex | +| `DJANGO_ALLOWED_HOSTS` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings allowed-hosts | +| `DJANGO_DEFAULT_FROM_EMAIL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | EMAIL https docs.djangoproject.com/en/dev/ref/settings default-from-email | +| `DJANGO_EMAIL_BACKEND` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py), [config/settings/local.py](file://config/settings/local.py) | EMAIL https docs.djangoproject.com/en/dev/ref/settings email-backend | +| `DJANGO_EMAIL_SUBJECT_PREFIX` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings email-subject-prefix | +| `DJANGO_READ_DOT_ENV_FILE` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - | +| `DJANGO_SECURE_CONTENT_TYPE_NOSNIFF` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/middleware x-content-type-options-nosniff | +| `DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-hsts-include-subdomains | +| `DJANGO_SECURE_HSTS_PRELOAD` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-hsts-preload | +| `DJANGO_SECURE_SSL_REDIRECT` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-ssl-redirect | +| `DJANGO_SERVER_EMAIL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings server-email | +| `DJANGO_SETTINGS_MODULE` | Django | Unknown | - | [config/wsgi.py](file://config/wsgi.py) | os.environ DJANGO_SETTINGS_MODULE = "config.settings.production" # noqa ERA001 | +| `OIDC_RP_CLIENT_ID` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - | +| `OIDC_RP_CLIENT_SECRET` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - | +| `POSTGRES_SSL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | - | +| `REDIS_HOST` | Django | Common | "redis" | [config/settings/base.py](file://config/settings/base.py) | - | +| `REDIS_URL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | - | +| `SCRAM_AUTH_METHOD` | Django | Common | "local" | [config/settings/base.py](file://config/settings/base.py) | Are you using local passwords or oidc? | +| `USE_DOCKER` | Django | Local | - | [config/settings/local.py](file://config/settings/local.py) | - | +| `BAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | A useful comment " VAR = os.getenv FOO # Same line comment " VAR2 = os.getenv BAR | +| `DEFAULT_VAR` | Other | Test | 'my_default' | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | Has default | +| `DJANGO_VAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | - | +| `ENV_VAR` | Other | Test | "env_def" | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | - | +| `FOO` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | A useful comment " VAR = os.getenv FOO # Same line comment " VAR2 = os.getenv BAR | +| `STANDARD_VAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | This is standard | +| `STRICT_VAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | - | +| `CELERY_BROKER_URL` | Scheduler | Test | - | [scheduler/tests/test_settings.py](file://scheduler/tests/test_settings.py) | - | +| `CELERY_RESULT_BACKEND` | Scheduler | Test | - | [scheduler/tests/test_settings.py](file://scheduler/tests/test_settings.py) | - | +| `DISABLE_PROCESS_UPDATES` | Scheduler | Test | - | [scheduler/tests/test_app.py](file://scheduler/tests/test_app.py) | Set the disable env var and then reload settings, then the app | +| `SCRAM_API_URL` | Scheduler | Test | - | [scheduler/tests/test_settings.py](file://scheduler/tests/test_settings.py) | - | +| `DEBUG` | Translator | Unknown | - | [translator/translator.py](file://translator/translator.py) | Here we setup a debugger if this is desired. This obviously should not be run in production | +| `SCRAM_EVENTS_URL` | Translator | Unknown | "ws://django:8000/ws/route_manager/translator_block/" | [translator/translator.py](file://translator/translator.py) | - | +| `SCRAM_HOSTNAME` | Translator | Unknown | "scram_hostname_not_set" | [translator/translator.py](file://translator/translator.py) | Must match the URL in asgi.py, and needs a trailing slash | diff --git a/pyproject.toml b/pyproject.toml index bbdea567..fe4f80ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [project] name = "SCRAM" version = "1.5.1" +requires-python = ">=3.12" # ==== pytest ==== [tool.pytest.ini_options] diff --git a/scripts/extract_env_vars.py b/scripts/extract_env_vars.py new file mode 100755 index 00000000..196634de --- /dev/null +++ b/scripts/extract_env_vars.py @@ -0,0 +1,341 @@ +"""Script to extract environment variables.""" + +# !/usr/bin/env uv run +import argparse +import difflib +import logging +import re +import sys +from pathlib import Path +from typing import Any + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Exclusion patterns for directories +EXCLUDE_DIRS = { + "venv", + ".venv", + ".git", + ".idea", + ".vscode", + ".pytest_cache", + ".ruff_cache", + "staticfiles", + "node_modules", + ".envs", + "__pycache__", +} + +# var_name, default_value from python files +PYTHON_ENV_PATTERNS = [ + ( + r'env(?:\.\w+)?\(\s*["\']([^"\']+)["\'](?:,\s*default=(?:get_random_secret_key\(\)|' + r'get_random_string\(50, allowed_chars="abcdefghijklmnopqrstuvwxyz0123456789"\)|' + r"[^,\)]+))?\s*\)" + ), + r'os\.getenv\(\s*["\']([^"\']+)["\'](?:,\s*([^,\)]+))?\s*\)', + r'os\.environ\.get\(\s*["\']([^"\']+)["\'](?:,\s*([^,\)]+))?\s*\)', + r'os\.environ\[\s*["\']([^"\']+)["\']\s*\]', +] + +# var_name, default_value from compose files +COMPOSE_ENV_PATTERN = r"\$\{([^}:-]+)(?::-([^}]*))?\}" + + +def extract_comment(lines: list[str], line_index: int) -> str: + """Pull comments from either the same line or right above to add context. + + Returns: + str: comment relevant to the envvar line + """ + # Check same line comments + current_line = lines[line_index] + if "#" in current_line: + comment = current_line.split("#", 1)[1].strip() + if comment: + return comment + + # Check lines above + comments_above = [] + # Start at the line above and continue up the lines until you find a line without a comment + # or you hit the beginning of the file + for i in range(line_index - 1, -1, -1): + prev_line = lines[i].strip() + if prev_line.startswith("#"): + comments_above.insert(0, prev_line.lstrip("#").strip()) + else: + break + + if comments_above: + return " ".join(comments_above) + return "" + + +def clean_comment(comment: str) -> str: + """Remove heavily repeated special characters which are mostly useless in docs. + + Returns: + str: a comment without special characters + """ + if not comment: + return "" + comment = re.sub(r"[^a-zA-Z0-9\s]{2,}", " ", comment) + comment = comment.strip(" #=-_*.") + comment = re.sub(r"\s+", " ", comment) + return comment.strip() + + +def extract_from_python(content: str, file_path: Path) -> dict[str, dict[str, Any]]: + """Extract environment variables from Python files. + + Returns: + dict: python environment variables + """ + vars_found: dict[str, dict[str, Any]] = {} + lines = content.splitlines() + for i, line in enumerate(lines): + for pattern in PYTHON_ENV_PATTERNS: + # finditer so that we have loop over the matches as dicts + # (match.group(1) is the var name, match.group(2) is the default value) + matches = re.finditer(pattern, line) + for match in matches: + var_name = match.group(1) + default_value = match.group(2) if len(match.groups()) > 1 else None + comment = clean_comment(extract_comment(lines, i)) + + if var_name not in vars_found: + vars_found[var_name] = {"default": default_value, "desc": comment, "file": {str(file_path)}} + # update comment if there wasnt one in previous findings of the var + elif comment and not vars_found[var_name]["desc"]: + vars_found[var_name]["desc"] = comment + return vars_found + + +def extract_from_compose(content: str, file_path: Path) -> dict[str, dict[str, Any]]: + """Extract environment variables from Compose files. + + Returns: + dict: compose environment variables + """ + vars_found: dict[str, dict[str, Any]] = {} + lines = content.splitlines() + for i, line in enumerate(lines): + matches = re.finditer(COMPOSE_ENV_PATTERN, line) + for match in matches: + var_name = match.group(1) + default_value = match.group(2) if len(match.groups()) > 1 else None + comment = clean_comment(extract_comment(lines, i)) + + if var_name not in vars_found: + vars_found[var_name] = {"default": default_value, "desc": comment, "file": {str(file_path)}} + elif comment and not vars_found[var_name]["desc"]: + vars_found[var_name]["desc"] = comment + return vars_found + + +def infer_environment(file_path: Path) -> str: + """Infer the environment from the file path. + + Returns: + str: the type of environment + """ + path_str = str(file_path) + if "production" in path_str: + return "Production" + if "local" in path_str: + return "Local" + if "test" in path_str: + return "Test" + if "settings/base" in path_str or "shared" in path_str or "common" in path_str or path_str == "compose.yml": + return "Common" + return "Unknown" + + +def get_service(file_path: Path) -> str: + """Determine the service name based on the file path. + + Returns: + str: the name of the service/app + """ + path_parts = Path(file_path).parts + if "config" in path_parts: + return "Django" + if "translator" in path_parts: + return "Translator" + if "scheduler" in path_parts: + return "Scheduler" + if "compose" in path_parts or Path(file_path).name.startswith("compose"): + return "Compose" + return "Other" + + +def parse_existing_docs(output_path: Path) -> dict[str, str]: + """Parse existing documentation to preserve manual descriptions. + + Returns: + dict: a dictionary with existing descriptions + """ + manual_descs: dict[str, str] = {} + if not output_path.exists(): + return manual_descs + + content = output_path.read_text(encoding="utf-8") + rows = re.findall(r"\| `([^`]+)` \| [^|]+ \| [^|]+ \| [^|]+ \| [^|]+ \| ([^|]*)\|", content) + for var, desc in rows: + clean_desc = clean_comment(desc.strip()) + if clean_desc and clean_desc not in {"-", "Manually added description"}: + manual_descs[var] = clean_desc + return manual_descs + + +def find_env_vars(root_dir: Path) -> dict[tuple[str, str], dict[str, Any]]: # noqa: C901 + """Scan the project directory for environment variables. + + Returns: + dict: a dictionary with all the environment variables + """ + all_vars: dict[tuple[str, str], dict[str, Any]] = {} + + for path in root_dir.rglob("*"): + if any(exclude in path.parts for exclude in EXCLUDE_DIRS): + continue + + rel_path = path.relative_to(root_dir) + + if path.suffix == ".py": + try: + content = path.read_text() + vars_found = extract_from_python(content, rel_path) + for var, info in vars_found.items(): + service = get_service(rel_path) + key = (var, service) + if key not in all_vars: + all_vars[key] = { + "envs": set(), + "default": info["default"], + "desc": info["desc"], + "file": set(), + } + all_vars[key]["envs"].add(infer_environment(rel_path)) + all_vars[key]["file"].update(info["file"]) + if info["desc"] and not all_vars[key]["desc"]: + all_vars[key]["desc"] = info["desc"] + except Exception: + logger.exception("Error reading %s", path) + + elif path.suffix in {".yml", ".yaml"} and "compose" in path.name: + try: + content = path.read_text() + vars_found = extract_from_compose(content, rel_path) + for var, info in vars_found.items(): + key = (var, "Compose") + if key not in all_vars: + all_vars[key] = { + "envs": set(), + "default": info["default"], + "desc": info["desc"], + "file": set(), + } + all_vars[key]["envs"].add(infer_environment(rel_path)) + all_vars[key]["file"].update(info["file"]) + if info["desc"] and not all_vars[key]["desc"]: + all_vars[key]["desc"] = info["desc"] + except Exception: + logger.exception("Error reading %s", path) + + return all_vars + + +def sort_by_service_and_name(item: tuple[tuple[str, str], dict[str, Any]]) -> tuple[str, str]: + """Sort by Service Name first, then Variable Name. + + Returns: + tuple: of the service name and variable name + """ + (var_name, service), _metadata = item + return service, var_name + + +def generate_markdown_content(all_vars: dict[tuple[str, str], dict[str, Any]], manual_descs: dict[str, str]) -> str: + """Generate the markdown content for the environment variables documentation. + + Returns: + str: the markdown content + """ + sorted_vars = sorted(all_vars.items(), key=sort_by_service_and_name) + + lines = [ + "# Environment Variables Reference", + "", + "To update, run `make update-env-docs`.", + "", + "| Variable | Service | Environments | Default | file | Description |", + "| --- | --- | --- | --- | --- | --- |", + ] + + for (var_name, service), data in sorted_vars: + if "Common" in data["envs"]: + envs = "Common" + elif len(data["envs"]) > 1: + envs = "Multiple" + else: + envs = ", ".join(sorted(data["envs"])) + + # Grab the default value or fall back to "-" + default = data["default"] if data["default"] else "-" + # Used to create a link to the file + file = ", ".join(f"[{f}](file://{f})" for f in sorted(data["file"])) + + # Use manual description if available, otherwise use code comments + description = manual_descs.get(var_name, data["desc"]) + if not description: + description = "-" + + lines.append(f"| `{var_name}` | {service} | {envs} | {default} | {file} | {description} |") + + return "\n".join(lines) + "\n" + + +def main() -> None: + """Main function to parse arguments and orchestrate the extraction.""" + parser = argparse.ArgumentParser(description="Extract environment variables from SCRAM project.") + parser.add_argument("--check", action="store_true", help="Check if documentation is up to date.") + args = parser.parse_args() + + root_dir = Path(__file__).resolve().parent.parent + output_path = root_dir / "docs/environment_variables.md" + + manual_descs = parse_existing_docs(output_path) + all_vars = find_env_vars(root_dir) + new_content = generate_markdown_content(all_vars, manual_descs) + + if args.check: + if output_path.exists(): + current_content = output_path.read_text() + if current_content == new_content: + logger.info("Documentation is up to date.") + return + logger.info("Documentation is out of date:\n") + diff = difflib.unified_diff( + current_content.splitlines(), + new_content.splitlines(), + fromfile="docs/environment_variables.md (Current)", + tofile="docs/environment_variables.md (Generated)", + lineterm="", + ) + logger.warning("\n".join(diff)) + sys.exit(1) + else: + logger.warning("Documentation file docs/environment_variables.md does not exist!") + sys.exit(1) + elif output_path.exists() and output_path.read_text() == new_content: + logger.info("Documentation is already up to date. No changes made.") + else: + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(new_content) + logger.info("Updated docs/environment_variables.md") + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_extract_env_vars.py b/scripts/tests/test_extract_env_vars.py new file mode 100644 index 00000000..5bf45745 --- /dev/null +++ b/scripts/tests/test_extract_env_vars.py @@ -0,0 +1,104 @@ +"""Define tests for our extract env vars script.""" + +from pathlib import Path + +from scripts.extract_env_vars import ( + clean_comment, + extract_comment, + extract_from_compose, + extract_from_python, + get_service, + infer_environment, +) + + +def test_extract_comment() -> None: + """Test extracting comments from adjacent or identical lines.""" + lines = [" # A useful comment", " VAR = os.getenv('FOO') # Same line comment", " VAR2 = os.getenv('BAR')"] + + assert extract_comment(lines, 1) == "Same line comment" + assert not extract_comment(lines, 2) + assert extract_comment(lines, 0) == "A useful comment" + + +def test_clean_comment() -> None: + """Test cleaning up special characters and excess whitespace from comments.""" + assert clean_comment(" # -- My comment ** ") == "My comment" + assert not clean_comment("") + assert not clean_comment(None) + + +def test_extract_from_python() -> None: + """Test extracting environment variables from Python source code strings.""" + content = """ + # This is standard + os.getenv("STANDARD_VAR") + + os.getenv('DEFAULT_VAR', 'my_default') # Has default + + os.environ.get("ENV_VAR", "env_def") + + os.environ["STRICT_VAR"] + + env.str("DJANGO_VAR", default="django_def") + """ + + dummy_path = Path("dummy.py") + result = extract_from_python(content, dummy_path) + + assert "STANDARD_VAR" in result + assert result["STANDARD_VAR"]["default"] is None + assert result["STANDARD_VAR"]["desc"] == "This is standard" + + assert "DEFAULT_VAR" in result + assert result["DEFAULT_VAR"]["default"] == "'my_default'" + assert result["DEFAULT_VAR"]["desc"] == "Has default" + + assert "ENV_VAR" in result + assert result["ENV_VAR"]["default"] == '"env_def"' + + assert "STRICT_VAR" in result + + assert "DJANGO_VAR" in result + assert result["DJANGO_VAR"]["default"] is None + + +def test_extract_from_compose() -> None: + """Test extracting environment variables and defaults from Compose YAML strings.""" + content = """ + services: + app: + environment: + - SIMPLE_VAR=${SIMPLE_VAR} # Simple description + - DEFAULT_VAR=${DEFAULT_VAR:-fallback_value} + """ + + dummy_path = Path("compose.yml") + result = extract_from_compose(content, dummy_path) + + assert "SIMPLE_VAR" in result + assert result["SIMPLE_VAR"]["default"] is None + assert result["SIMPLE_VAR"]["desc"] == "Simple description" + + assert "DEFAULT_VAR" in result + assert result["DEFAULT_VAR"]["default"] == "fallback_value" + + +def test_infer_environment() -> None: + """Test inferring the environment label based on specific file paths.""" + assert infer_environment(Path("config/settings/production.py")) == "Production" + assert infer_environment(Path("config/settings/local.py")) == "Local" + assert infer_environment(Path("tests/test_something.py")) == "Test" + assert infer_environment(Path("config/settings/base.py")) == "Common" + assert infer_environment(Path("translator/shared.py")) == "Common" + assert infer_environment(Path("compose.yml")) == "Common" + assert infer_environment(Path("random_file.py")) == "Unknown" + + +def test_get_service() -> None: + """Test identifying the service name based on directory structure or file name.""" + assert get_service(Path("config/settings.py")) == "Django" + assert get_service(Path("translator/app.py")) == "Translator" + assert get_service(Path("scheduler/tasks.py")) == "Scheduler" + assert get_service(Path("compose.override.yml")) == "Compose" + assert get_service(Path("scripts/extract_env_vars.py")) == "Other" From efa9acad41cdd80bc16d98f74e7c2a5aa3cd2a2e Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Tue, 3 Mar 2026 19:27:35 -0600 Subject: [PATCH 2/8] refactor(ignore): dont bring in env vars from test files, it just muddies the water and is of little value --- scripts/extract_env_vars.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/extract_env_vars.py b/scripts/extract_env_vars.py index 196634de..aba5250b 100755 --- a/scripts/extract_env_vars.py +++ b/scripts/extract_env_vars.py @@ -1,6 +1,6 @@ +# !/usr/bin/env uv run """Script to extract environment variables.""" -# !/usr/bin/env uv run import argparse import difflib import logging @@ -15,6 +15,10 @@ # Exclusion patterns for directories EXCLUDE_DIRS = { "venv", + "scripts/tests", + "scram/route_manager/tests", + "translator/tests", + "scheduler/tests", ".venv", ".git", ".idea", From 4ed43bbecf32bc7933d6d3ce470f216586f36783 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Tue, 3 Mar 2026 19:30:33 -0600 Subject: [PATCH 3/8] fix(comments): remove ugly comments that were picked up --- config/wsgi.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/wsgi.py b/config/wsgi.py index 73ed363b..1ebc25c0 100644 --- a/config/wsgi.py +++ b/config/wsgi.py @@ -21,15 +21,18 @@ # This allows easy placement of apps within the interior # scram directory. + ROOT_DIR = Path(__file__).resolve(strict=True).parent.parent sys.path.append(str(ROOT_DIR / "scram")) # We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks # if running multiple sites in the same mod_wsgi process. To fix this, use # mod_wsgi daemon mode with each site in its own daemon process, or use # os.environ["DJANGO_SETTINGS_MODULE"] = "config.settings.production" # noqa ERA001 + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production") # This application object is used by any WSGI server configured to use this # file. This includes Django's development server, if the WSGI_APPLICATION # setting points here. + application = get_wsgi_application() From 599bcd6b8ba1afdee1ada3a20d52fe917186f00b Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Tue, 3 Mar 2026 19:48:24 -0600 Subject: [PATCH 4/8] style(ruff): idk formatting stuff --- docs/environment_variables.md | 14 ++------------ scripts/extract_env_vars.py | 21 ++++++++++++--------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 6d36dbd8..95a64423 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -36,7 +36,7 @@ To update, run `make update-env-docs`. | `DJANGO_SECURE_HSTS_PRELOAD` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-hsts-preload | | `DJANGO_SECURE_SSL_REDIRECT` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-ssl-redirect | | `DJANGO_SERVER_EMAIL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings server-email | -| `DJANGO_SETTINGS_MODULE` | Django | Unknown | - | [config/wsgi.py](file://config/wsgi.py) | os.environ DJANGO_SETTINGS_MODULE = "config.settings.production" # noqa ERA001 | +| `DJANGO_SETTINGS_MODULE` | Django | Unknown | "config.settings.local" | [config/asgi.py](file://config/asgi.py), [config/wsgi.py](file://config/wsgi.py) | If DJANGO_SETTINGS_MODULE is unset, default to the local settings | | `OIDC_RP_CLIENT_ID` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - | | `OIDC_RP_CLIENT_SECRET` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - | | `POSTGRES_SSL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | - | @@ -44,17 +44,7 @@ To update, run `make update-env-docs`. | `REDIS_URL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | - | | `SCRAM_AUTH_METHOD` | Django | Common | "local" | [config/settings/base.py](file://config/settings/base.py) | Are you using local passwords or oidc? | | `USE_DOCKER` | Django | Local | - | [config/settings/local.py](file://config/settings/local.py) | - | -| `BAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | A useful comment " VAR = os.getenv FOO # Same line comment " VAR2 = os.getenv BAR | -| `DEFAULT_VAR` | Other | Test | 'my_default' | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | Has default | -| `DJANGO_VAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | - | -| `ENV_VAR` | Other | Test | "env_def" | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | - | -| `FOO` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | A useful comment " VAR = os.getenv FOO # Same line comment " VAR2 = os.getenv BAR | -| `STANDARD_VAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | This is standard | -| `STRICT_VAR` | Other | Test | - | [scripts/tests/test_extract_env_vars.py](file://scripts/tests/test_extract_env_vars.py) | - | -| `CELERY_BROKER_URL` | Scheduler | Test | - | [scheduler/tests/test_settings.py](file://scheduler/tests/test_settings.py) | - | -| `CELERY_RESULT_BACKEND` | Scheduler | Test | - | [scheduler/tests/test_settings.py](file://scheduler/tests/test_settings.py) | - | -| `DISABLE_PROCESS_UPDATES` | Scheduler | Test | - | [scheduler/tests/test_app.py](file://scheduler/tests/test_app.py) | Set the disable env var and then reload settings, then the app | -| `SCRAM_API_URL` | Scheduler | Test | - | [scheduler/tests/test_settings.py](file://scheduler/tests/test_settings.py) | - | +| `DJANGO_SETTINGS_MODULE` | Other | Unknown | "config.settings.local" | [manage.py](file://manage.py) | - | | `DEBUG` | Translator | Unknown | - | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | Here we setup a debugger if this is desired. This obviously should not be run in production | | `SCRAM_EVENTS_URL` | Translator | Unknown | "ws://django:8000/ws/route_manager/translator_block/" | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | - | | `SCRAM_HOSTNAME` | Translator | Unknown | "scram_hostname_not_set" | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | Must match the URL in asgi.py, and needs a trailing slash | diff --git a/scripts/extract_env_vars.py b/scripts/extract_env_vars.py index aba5250b..f7473a67 100755 --- a/scripts/extract_env_vars.py +++ b/scripts/extract_env_vars.py @@ -40,6 +40,7 @@ ), r'os\.getenv\(\s*["\']([^"\']+)["\'](?:,\s*([^,\)]+))?\s*\)', r'os\.environ\.get\(\s*["\']([^"\']+)["\'](?:,\s*([^,\)]+))?\s*\)', + r'os\.environ\.setdefault\(\s*["\']([^"\']+)["\'](?:,\s*([^,\)]+))?\s*\)', r'os\.environ\[\s*["\']([^"\']+)["\']\s*\]', ] @@ -99,6 +100,9 @@ def extract_from_python(content: str, file_path: Path) -> dict[str, dict[str, An vars_found: dict[str, dict[str, Any]] = {} lines = content.splitlines() for i, line in enumerate(lines): + # Skip pure comment lines (commented-out code can match patterns but isn't real usage) + if line.strip().startswith("#"): + continue for pattern in PYTHON_ENV_PATTERNS: # finditer so that we have loop over the matches as dicts # (match.group(1) is the var name, match.group(2) is the default value) @@ -174,22 +178,22 @@ def get_service(file_path: Path) -> str: return "Other" -def parse_existing_docs(output_path: Path) -> dict[str, str]: +def parse_existing_docs(output_path: Path) -> dict[tuple[str, str], str]: """Parse existing documentation to preserve manual descriptions. Returns: dict: a dictionary with existing descriptions """ - manual_descs: dict[str, str] = {} + manual_descs: dict[tuple[str, str], str] = {} if not output_path.exists(): return manual_descs content = output_path.read_text(encoding="utf-8") - rows = re.findall(r"\| `([^`]+)` \| [^|]+ \| [^|]+ \| [^|]+ \| [^|]+ \| ([^|]*)\|", content) - for var, desc in rows: + rows = re.findall(r"\| `([^`]+)` \| ([^|]+) \| [^|]+ \| [^|]+ \| [^|]+ \| ([^|]*)\|", content) + for var, service, desc in rows: clean_desc = clean_comment(desc.strip()) if clean_desc and clean_desc not in {"-", "Manually added description"}: - manual_descs[var] = clean_desc + manual_descs[var, service.strip()] = clean_desc return manual_descs @@ -202,10 +206,9 @@ def find_env_vars(root_dir: Path) -> dict[tuple[str, str], dict[str, Any]]: # n all_vars: dict[tuple[str, str], dict[str, Any]] = {} for path in root_dir.rglob("*"): - if any(exclude in path.parts for exclude in EXCLUDE_DIRS): - continue - rel_path = path.relative_to(root_dir) + if any(exclude in path.parts or exclude in str(rel_path) for exclude in EXCLUDE_DIRS): + continue if path.suffix == ".py": try: @@ -292,7 +295,7 @@ def generate_markdown_content(all_vars: dict[tuple[str, str], dict[str, Any]], m file = ", ".join(f"[{f}](file://{f})" for f in sorted(data["file"])) # Use manual description if available, otherwise use code comments - description = manual_descs.get(var_name, data["desc"]) + description = manual_descs.get((var_name, service), data["desc"]) if not description: description = "-" From 082907802b11eb36e2e4ba735c9c8cf906772d34 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Tue, 3 Mar 2026 23:46:33 -0600 Subject: [PATCH 5/8] fix(regex): we were missing a capture group so it always returned None which hid the bug --- docs/environment_variables.md | 26 +++++++++++++------------- scripts/extract_env_vars.py | 2 +- scripts/tests/test_extract_env_vars.py | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 95a64423..6c88e3cc 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -22,28 +22,28 @@ To update, run `make update-env-docs`. | `SCRAM_V6_GATEWAY` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | | `SCRAM_V6_SUBNET` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | | `TRANSLATOR_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | -| `CONN_MAX_AGE` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | noqa F405 | +| `CONN_MAX_AGE` | Django | Production | 60 | [config/settings/production.py](file://config/settings/production.py) | noqa F405 | | `DATABASE_URL` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py), [config/settings/production.py](file://config/settings/production.py) | DATABASES https docs.djangoproject.com/en/dev/ref/settings databases | | `DEBUG` | Django | Unknown | - | [config/asgi.py](file://config/asgi.py) | Here we setup a debugger if this is desired. This obviously should not be run in production | | `DJANGO_ADMIN_URL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | ADMIN Django Admin URL regex | -| `DJANGO_ALLOWED_HOSTS` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings allowed-hosts | -| `DJANGO_DEFAULT_FROM_EMAIL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | EMAIL https docs.djangoproject.com/en/dev/ref/settings default-from-email | -| `DJANGO_EMAIL_BACKEND` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py), [config/settings/local.py](file://config/settings/local.py) | EMAIL https docs.djangoproject.com/en/dev/ref/settings email-backend | -| `DJANGO_EMAIL_SUBJECT_PREFIX` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings email-subject-prefix | -| `DJANGO_READ_DOT_ENV_FILE` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - | -| `DJANGO_SECURE_CONTENT_TYPE_NOSNIFF` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/middleware x-content-type-options-nosniff | -| `DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-hsts-include-subdomains | -| `DJANGO_SECURE_HSTS_PRELOAD` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-hsts-preload | -| `DJANGO_SECURE_SSL_REDIRECT` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-ssl-redirect | -| `DJANGO_SERVER_EMAIL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings server-email | +| `DJANGO_ALLOWED_HOSTS` | Django | Production | ["django"] | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings allowed-hosts | +| `DJANGO_DEFAULT_FROM_EMAIL` | Django | Production | "SCRAM " | [config/settings/production.py](file://config/settings/production.py) | EMAIL https docs.djangoproject.com/en/dev/ref/settings default-from-email | +| `DJANGO_EMAIL_BACKEND` | Django | Common | "django.core.mail.backends.console.EmailBackend" | [config/settings/base.py](file://config/settings/base.py), [config/settings/local.py](file://config/settings/local.py) | EMAIL https docs.djangoproject.com/en/dev/ref/settings email-backend | +| `DJANGO_EMAIL_SUBJECT_PREFIX` | Django | Production | "[SCRAM]" | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings email-subject-prefix | +| `DJANGO_READ_DOT_ENV_FILE` | Django | Common | False | [config/settings/base.py](file://config/settings/base.py) | - | +| `DJANGO_SECURE_CONTENT_TYPE_NOSNIFF` | Django | Production | True | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/middleware x-content-type-options-nosniff | +| `DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS` | Django | Production | True | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-hsts-include-subdomains | +| `DJANGO_SECURE_HSTS_PRELOAD` | Django | Production | True | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-hsts-preload | +| `DJANGO_SECURE_SSL_REDIRECT` | Django | Production | True | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings secure-ssl-redirect | +| `DJANGO_SERVER_EMAIL` | Django | Production | DEFAULT_FROM_EMAIL | [config/settings/production.py](file://config/settings/production.py) | https docs.djangoproject.com/en/dev/ref/settings server-email | | `DJANGO_SETTINGS_MODULE` | Django | Unknown | "config.settings.local" | [config/asgi.py](file://config/asgi.py), [config/wsgi.py](file://config/wsgi.py) | If DJANGO_SETTINGS_MODULE is unset, default to the local settings | | `OIDC_RP_CLIENT_ID` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - | | `OIDC_RP_CLIENT_SECRET` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py) | - | -| `POSTGRES_SSL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | - | +| `POSTGRES_SSL` | Django | Production | True | [config/settings/production.py](file://config/settings/production.py) | - | | `REDIS_HOST` | Django | Common | "redis" | [config/settings/base.py](file://config/settings/base.py) | - | | `REDIS_URL` | Django | Production | - | [config/settings/production.py](file://config/settings/production.py) | - | | `SCRAM_AUTH_METHOD` | Django | Common | "local" | [config/settings/base.py](file://config/settings/base.py) | Are you using local passwords or oidc? | -| `USE_DOCKER` | Django | Local | - | [config/settings/local.py](file://config/settings/local.py) | - | +| `USE_DOCKER` | Django | Local | "no" | [config/settings/local.py](file://config/settings/local.py) | - | | `DJANGO_SETTINGS_MODULE` | Other | Unknown | "config.settings.local" | [manage.py](file://manage.py) | - | | `DEBUG` | Translator | Unknown | - | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | Here we setup a debugger if this is desired. This obviously should not be run in production | | `SCRAM_EVENTS_URL` | Translator | Unknown | "ws://django:8000/ws/route_manager/translator_block/" | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | - | diff --git a/scripts/extract_env_vars.py b/scripts/extract_env_vars.py index f7473a67..c3a3abc0 100755 --- a/scripts/extract_env_vars.py +++ b/scripts/extract_env_vars.py @@ -36,7 +36,7 @@ ( r'env(?:\.\w+)?\(\s*["\']([^"\']+)["\'](?:,\s*default=(?:get_random_secret_key\(\)|' r'get_random_string\(50, allowed_chars="abcdefghijklmnopqrstuvwxyz0123456789"\)|' - r"[^,\)]+))?\s*\)" + r"([^,\)]+)))?\s*\)" ), r'os\.getenv\(\s*["\']([^"\']+)["\'](?:,\s*([^,\)]+))?\s*\)', r'os\.environ\.get\(\s*["\']([^"\']+)["\'](?:,\s*([^,\)]+))?\s*\)', diff --git a/scripts/tests/test_extract_env_vars.py b/scripts/tests/test_extract_env_vars.py index 5bf45745..3846f71b 100644 --- a/scripts/tests/test_extract_env_vars.py +++ b/scripts/tests/test_extract_env_vars.py @@ -60,7 +60,7 @@ def test_extract_from_python() -> None: assert "STRICT_VAR" in result assert "DJANGO_VAR" in result - assert result["DJANGO_VAR"]["default"] is None + assert result["DJANGO_VAR"]["default"] == '"django_def"' def test_extract_from_compose() -> None: From 4ffc13fe6e4576afd6f10dff8dc1704f12cb438d Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Tue, 3 Mar 2026 23:51:24 -0600 Subject: [PATCH 6/8] fix(typo): accidentally deleted a --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 7f887458..43f60324 100644 --- a/Makefile +++ b/Makefile @@ -175,6 +175,7 @@ copy-libs: @docker compose cp translator:/app/attribute_pb2_grpc.py translator/ @docker compose cp translator:/app/capability_pb2.py translator/ @docker compose cp translator:/app/capability_pb2.pyi translator/ + @docker compose cp translator:/app/capability_pb2_grpc.py translator/ ## update-env-docs: update environment variable documentation append CHECK=true to get a diff if not up to date .Phony: update-env-docs From ee7c063001f562a9d73491128ed97b9aff672258 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Wed, 4 Mar 2026 14:50:42 -0600 Subject: [PATCH 7/8] refactor(env choosing): use a more robust method for discovering environment and clean up some code --- docs/environment_variables.md | 26 ++++---- scripts/extract_env_vars.py | 119 ++++++++++++++++++---------------- 2 files changed, 76 insertions(+), 69 deletions(-) diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 6c88e3cc..6bb5f1f0 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -4,16 +4,16 @@ To update, run `make update-env-docs`. | Variable | Service | Environments | Default | file | Description | | --- | --- | --- | --- | --- | --- | -| `CELERY_BEAT_REPLICAS` | Compose | Common | 0 | [compose.yml](file://compose.yml) | - | -| `CELERY_WORKER_REPLICAS` | Compose | Common | 0 | [compose.yml](file://compose.yml) | - | -| `DEBUG` | Compose | Multiple | - | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.yml](file://compose.override.yml) | Here we setup a debugger if this is desired. This obviously should not be run in production | -| `DJANGO_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | -| `DOCS_PORT` | Compose | Multiple | 8888 | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.yml](file://compose.override.yml) | - | -| `FLOWER_REPLICAS` | Compose | Common | 0 | [compose.yml](file://compose.yml) | - | -| `GOBGP_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | +| `CELERY_BEAT_REPLICAS` | Compose | Unknown | 0 | [compose.yml](file://compose.yml) | - | +| `CELERY_WORKER_REPLICAS` | Compose | Unknown | 0 | [compose.yml](file://compose.yml) | - | +| `DEBUG` | Compose | Local | - | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.yml](file://compose.override.yml) | This can be set to either `debugpy` or `pycharm-pydevd` currently | +| `DJANGO_REPLICAS` | Compose | Unknown | 1 | [compose.yml](file://compose.yml) | - | +| `DOCS_PORT` | Compose | Local | 8888 | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.yml](file://compose.override.yml) | - | +| `FLOWER_REPLICAS` | Compose | Unknown | 0 | [compose.yml](file://compose.yml) | - | +| `GOBGP_REPLICAS` | Compose | Unknown | 1 | [compose.yml](file://compose.yml) | - | | `HOSTNAME` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | -| `POSTGRES_ENABLED` | Compose | Common | 1 | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.production.yml](file://compose.override.production.yml), [compose.override.yml](file://compose.override.yml), [compose.yml](file://compose.yml) | - | -| `REDIS_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | +| `POSTGRES_ENABLED` | Compose | Multiple | 1 | [compose.override.local.yml](file://compose.override.local.yml), [compose.override.production.yml](file://compose.override.production.yml), [compose.override.yml](file://compose.override.yml), [compose.yml](file://compose.yml) | - | +| `REDIS_REPLICAS` | Compose | Unknown | 1 | [compose.yml](file://compose.yml) | - | | `SCRAM_PEERING_IFACE` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | | `SCRAM_V4_ADDRESS` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | | `SCRAM_V4_GATEWAY` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | @@ -21,7 +21,7 @@ To update, run `make update-env-docs`. | `SCRAM_V6_ADDRESS` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | | `SCRAM_V6_GATEWAY` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | | `SCRAM_V6_SUBNET` | Compose | Production | - | [compose.override.production.yml](file://compose.override.production.yml) | - | -| `TRANSLATOR_REPLICAS` | Compose | Common | 1 | [compose.yml](file://compose.yml) | - | +| `TRANSLATOR_REPLICAS` | Compose | Unknown | 1 | [compose.yml](file://compose.yml) | - | | `CONN_MAX_AGE` | Django | Production | 60 | [config/settings/production.py](file://config/settings/production.py) | noqa F405 | | `DATABASE_URL` | Django | Common | - | [config/settings/base.py](file://config/settings/base.py), [config/settings/production.py](file://config/settings/production.py) | DATABASES https docs.djangoproject.com/en/dev/ref/settings databases | | `DEBUG` | Django | Unknown | - | [config/asgi.py](file://config/asgi.py) | Here we setup a debugger if this is desired. This obviously should not be run in production | @@ -45,6 +45,6 @@ To update, run `make update-env-docs`. | `SCRAM_AUTH_METHOD` | Django | Common | "local" | [config/settings/base.py](file://config/settings/base.py) | Are you using local passwords or oidc? | | `USE_DOCKER` | Django | Local | "no" | [config/settings/local.py](file://config/settings/local.py) | - | | `DJANGO_SETTINGS_MODULE` | Other | Unknown | "config.settings.local" | [manage.py](file://manage.py) | - | -| `DEBUG` | Translator | Unknown | - | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | Here we setup a debugger if this is desired. This obviously should not be run in production | -| `SCRAM_EVENTS_URL` | Translator | Unknown | "ws://django:8000/ws/route_manager/translator_block/" | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | - | -| `SCRAM_HOSTNAME` | Translator | Unknown | "scram_hostname_not_set" | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | Must match the URL in asgi.py, and needs a trailing slash | +| `DEBUG` | Translator | Production | - | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | Here we setup a debugger if this is desired. This obviously should not be run in production | +| `SCRAM_EVENTS_URL` | Translator | Production | "ws://django:8000/ws/route_manager/translator_block/" | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | - | +| `SCRAM_HOSTNAME` | Translator | Production | "scram_hostname_not_set" | [translator/src/translator/translator.py](file://translator/src/translator/translator.py) | Must match the URL in asgi.py, and needs a trailing slash | diff --git a/scripts/extract_env_vars.py b/scripts/extract_env_vars.py index c3a3abc0..37023758 100755 --- a/scripts/extract_env_vars.py +++ b/scripts/extract_env_vars.py @@ -6,7 +6,7 @@ import logging import re import sys -from pathlib import Path +from pathlib import Path, PurePath from typing import Any logging.basicConfig(level=logging.INFO) @@ -47,6 +47,16 @@ # var_name, default_value from compose files COMPOSE_ENV_PATTERN = r"\$\{([^}:-]+)(?::-([^}]*))?\}" +ENVIRONMENT_MAP = { + "test": "Test", + "production": "Production", + "translator": "Production", + "scheduler": "Production", + "local": "Local", + "shared": "Common", + "common": "Common", +} + def extract_comment(lines: list[str], line_index: int) -> str: """Pull comments from either the same line or right above to add context. @@ -148,15 +158,20 @@ def infer_environment(file_path: Path) -> str: Returns: str: the type of environment """ - path_str = str(file_path) - if "production" in path_str: - return "Production" - if "local" in path_str: - return "Local" - if "test" in path_str: - return "Test" - if "settings/base" in path_str or "shared" in path_str or "common" in path_str or path_str == "compose.yml": + p = PurePath(file_path) + path_parts = {part.lower() for part in p.parts} + filename = p.name.lower() + + for part, env in ENVIRONMENT_MAP.items(): + if part in path_parts or part in filename: + return env + + if "settings" in path_parts and "base" in path_parts: + return "Common" + + if "base" in filename: return "Common" + return "Unknown" @@ -166,7 +181,7 @@ def get_service(file_path: Path) -> str: Returns: str: the name of the service/app """ - path_parts = Path(file_path).parts + path_parts = PurePath(file_path).parts if "config" in path_parts: return "Django" if "translator" in path_parts: @@ -197,7 +212,29 @@ def parse_existing_docs(output_path: Path) -> dict[tuple[str, str], str]: return manual_descs -def find_env_vars(root_dir: Path) -> dict[tuple[str, str], dict[str, Any]]: # noqa: C901 +def _accumulate_vars( + all_vars: dict, + vars_found: dict, + rel_path: Path, + service: str, +) -> None: + env = infer_environment(rel_path) + for var, info in vars_found.items(): + key = (var, service) + if key not in all_vars: + all_vars[key] = { + "envs": set(), + "default": info["default"], + "desc": info["desc"], + "file": set(), + } + all_vars[key]["envs"].add(env) + all_vars[key]["file"].update(info["file"]) + if info["desc"] and not all_vars[key]["desc"]: + all_vars[key]["desc"] = info["desc"] + + +def find_env_vars(root_dir: Path) -> dict[tuple[str, str], dict[str, Any]]: """Scan the project directory for environment variables. Returns: @@ -210,46 +247,17 @@ def find_env_vars(root_dir: Path) -> dict[tuple[str, str], dict[str, Any]]: # n if any(exclude in path.parts or exclude in str(rel_path) for exclude in EXCLUDE_DIRS): continue - if path.suffix == ".py": - try: - content = path.read_text() - vars_found = extract_from_python(content, rel_path) - for var, info in vars_found.items(): - service = get_service(rel_path) - key = (var, service) - if key not in all_vars: - all_vars[key] = { - "envs": set(), - "default": info["default"], - "desc": info["desc"], - "file": set(), - } - all_vars[key]["envs"].add(infer_environment(rel_path)) - all_vars[key]["file"].update(info["file"]) - if info["desc"] and not all_vars[key]["desc"]: - all_vars[key]["desc"] = info["desc"] - except Exception: - logger.exception("Error reading %s", path) - - elif path.suffix in {".yml", ".yaml"} and "compose" in path.name: - try: - content = path.read_text() - vars_found = extract_from_compose(content, rel_path) - for var, info in vars_found.items(): - key = (var, "Compose") - if key not in all_vars: - all_vars[key] = { - "envs": set(), - "default": info["default"], - "desc": info["desc"], - "file": set(), - } - all_vars[key]["envs"].add(infer_environment(rel_path)) - all_vars[key]["file"].update(info["file"]) - if info["desc"] and not all_vars[key]["desc"]: - all_vars[key]["desc"] = info["desc"] - except Exception: - logger.exception("Error reading %s", path) + try: + if path.suffix == ".py": + vars_found = extract_from_python(path.read_text(), rel_path) + _accumulate_vars(all_vars, vars_found, rel_path, get_service(rel_path)) + + elif path.suffix in {".yml", ".yaml"} and "compose" in path.name: + vars_found = extract_from_compose(path.read_text(), rel_path) + _accumulate_vars(all_vars, vars_found, rel_path, "Compose") + + except Exception: + logger.exception("Error reading %s", path) return all_vars @@ -282,19 +290,18 @@ def generate_markdown_content(all_vars: dict[tuple[str, str], dict[str, Any]], m ] for (var_name, service), data in sorted_vars: + real_envs = data["envs"] - {"Unknown"} + if "Common" in data["envs"]: envs = "Common" - elif len(data["envs"]) > 1: + elif len(real_envs) > 1: envs = "Multiple" else: - envs = ", ".join(sorted(data["envs"])) + envs = real_envs.pop() if real_envs else "Unknown" - # Grab the default value or fall back to "-" default = data["default"] if data["default"] else "-" - # Used to create a link to the file file = ", ".join(f"[{f}](file://{f})" for f in sorted(data["file"])) - # Use manual description if available, otherwise use code comments description = manual_descs.get((var_name, service), data["desc"]) if not description: description = "-" From 5dbf07a641531ca6f6359ec459d52ea99fc44402 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 6 Mar 2026 00:58:12 -0600 Subject: [PATCH 8/8] refactor(tests): the order of the map matters. we were catching the translator = Production branch before the shared one for translator/shared.py so tests failed --- scripts/extract_env_vars.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/extract_env_vars.py b/scripts/extract_env_vars.py index 37023758..8e335ea1 100755 --- a/scripts/extract_env_vars.py +++ b/scripts/extract_env_vars.py @@ -48,13 +48,13 @@ COMPOSE_ENV_PATTERN = r"\$\{([^}:-]+)(?::-([^}]*))?\}" ENVIRONMENT_MAP = { + "shared": "Common", + "common": "Common", "test": "Test", "production": "Production", "translator": "Production", "scheduler": "Production", "local": "Local", - "shared": "Common", - "common": "Common", }