diff --git a/docs/web/.gitignore b/docs/web/.gitignore index 8214b3b2..b1b65bde 100644 --- a/docs/web/.gitignore +++ b/docs/web/.gitignore @@ -1,5 +1,5 @@ # Output generated by `hugo` public/ -# Generated by update_docs.py, based the content in /docker/ and additional locations +# Generated by update-docs.py, based on the content in /docker/ and additional locations src/content/ diff --git a/docs/web/Dockerfile b/docs/web/Dockerfile index 94589649..c2ef1302 100644 --- a/docs/web/Dockerfile +++ b/docs/web/Dockerfile @@ -8,7 +8,7 @@ COPY docs/web/requirements.txt requirements.txt RUN pip install --no-cache-dir -r requirements.txt # Note: Add parameter for verbose output: --verbose -RUN --mount=type=bind,ro,source=.,target=/repo python -m docs.web.update_docs --repository-path /repo --output-content-path /src/content +RUN --mount=type=bind,ro,source=.,target=/repo /repo/docs/web/update-docs.py --repository-path /repo --output-content-path /src/content ##################################################################### # Build Stage # diff --git a/docs/web/README.md b/docs/web/README.md index 3f9471e5..add8b54b 100644 --- a/docs/web/README.md +++ b/docs/web/README.md @@ -5,7 +5,7 @@ A website is built with the [Hugo](https://gohugo.io/) static site generator, us ## Development -**Build process:** The Markdown and Docker Compose files are collected and converted by `update_docs.py`, then `hugo` build is executed in a Docker container, producing a container image with `nginx` serving the static website. +**Build process:** The Markdown and Docker Compose files are collected and converted by `update-docs.py`, then `hugo` build is executed in a Docker container, producing a container image with `nginx` serving the static website. Run `task docs:deploy` to build and locally deploy (using `docker/tools/homelab-docs.yaml`) the site. diff --git a/docs/web/Taskfile.web.yaml b/docs/web/Taskfile.web.yaml index 6c4696e2..0fbc8da4 100644 --- a/docs/web/Taskfile.web.yaml +++ b/docs/web/Taskfile.web.yaml @@ -53,3 +53,8 @@ tasks: desc: Clear generated documentation content cmds: - rm -rf docs/web/public/ + + create-service-list: + desc: Create a YAML file describing the services + cmds: + - docs/web/export-services.py --output-file docs/services.yaml diff --git a/docs/web/__init__.py b/docs/web/__init__.py deleted file mode 100644 index 37f4c45c..00000000 --- a/docs/web/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Documentation website package. -""" diff --git a/docs/web/docker_scanner.py b/docs/web/docker_scanner.py index fde2a9df..a2ea6be7 100644 --- a/docs/web/docker_scanner.py +++ b/docs/web/docker_scanner.py @@ -4,7 +4,7 @@ import os from pathlib import Path -from .compose_processor import ComposeFileProcessor +from compose_processor import ComposeFileProcessor class DockerComposeScanner: diff --git a/docs/web/export_services.py b/docs/web/export-services.py similarity index 75% rename from docs/web/export_services.py rename to docs/web/export-services.py index cea78832..b0dc9f60 100755 --- a/docs/web/export_services.py +++ b/docs/web/export-services.py @@ -9,9 +9,8 @@ from pathlib import Path import yaml - -from .docker_scanner import DockerComposeScanner -from .git_utils import get_git_root +from docker_scanner import DockerComposeScanner +from git_utils import get_git_root def str_presenter(dumper, data): @@ -24,11 +23,11 @@ def str_presenter(dumper, data): def export_services(repository_path, output_file, docker_path="docker", verbose=False): - """Export all Docker Compose services to a YAML file. + """Export all Docker Compose services to a YAML file or stdout. Args: repository_path: Path to the repository root - output_file: Path to the output YAML file + output_file: Path to the output YAML file, or None for stdout docker_path: Relative path to docker directory (default: "docker") verbose: Enable verbose logging """ @@ -69,10 +68,20 @@ def export_services(repository_path, output_file, docker_path="docker", verbose= output_data["services"].append(service_data) - # Write to YAML file - logger.info(f"Writing {len(services)} services to {output_file}") - with open(output_file, "w") as f: - yaml.dump(output_data, f, width=math.inf, default_flow_style=False, sort_keys=False, allow_unicode=True, Dumper=yaml.Dumper) + # Write to YAML file or stdout + if output_file is not None: + logger.info(f"Writing {len(services)} services to {output_file}") + with open(output_file, "w") as stream: + yaml.dump( + output_data, stream, width=math.inf, default_flow_style=False, + sort_keys=False, allow_unicode=True, indent=2, Dumper=yaml.Dumper + ) + else: + logger.info("Writing services to stdout") + yaml.dump( + output_data, sys.stdout, width=math.inf, default_flow_style=False, + sort_keys=False, allow_unicode=True, indent=2, Dumper=yaml.Dumper + ) logger.info("Export complete") @@ -90,7 +99,7 @@ def export_services(repository_path, output_file, docker_path="docker", verbose= parser.add_argument( "--output-file", type=str, - help="Specify the output YAML file path" + help="Specify the output YAML file path (defaults to stdout if omitted)" ) parser.add_argument( "--docker-path", @@ -103,8 +112,8 @@ def export_services(repository_path, output_file, docker_path="docker", verbose= # Get repository path repository_path = Path(args.repository_path) if args.repository_path else Path(get_git_root()) - # Default output file if not specified - output_file = Path(args.output_file) if args.output_file else repository_path / "services.yaml" + # Default output to stdout if not specified + output_file = Path(args.output_file) if args.output_file else None # Export services try: diff --git a/docs/web/update_docs.py b/docs/web/update-docs.py similarity index 98% rename from docs/web/update_docs.py rename to docs/web/update-docs.py index 8b80f91f..974c044e 100755 --- a/docs/web/update_docs.py +++ b/docs/web/update-docs.py @@ -8,10 +8,9 @@ from pathlib import Path import yaml - -from .docker_scanner import DockerComposeScanner -from .git_utils import get_git_root -from .link_processor import LinkProcessor +from docker_scanner import DockerComposeScanner +from git_utils import get_git_root +from link_processor import LinkProcessor class DocsProcessor: diff --git a/scripts/infra-mcp/tools/get_container_tags.py b/scripts/infra-mcp/tools/get_container_tags.py index d7f5c6d2..419c9b61 100755 --- a/scripts/infra-mcp/tools/get_container_tags.py +++ b/scripts/infra-mcp/tools/get_container_tags.py @@ -35,8 +35,170 @@ def _parse_arch(self, arch: str) -> tuple[str, str]: arch_part = parts[1] if len(parts) > 1 else 'amd64' return os_part, arch_part - def get_docker_hub_tags(self, image_name: str, limit: int = 10, architecture: str = "linux/amd64") -> list[dict[str, Any]]: - """Query Docker Hub for image tags with timestamp information.""" + def _parse_version(self, tag_name: str) -> tuple[int, ...] | None: + """ + Extract version numbers from tag name. + + Handles various tag formats: + - "18.1", "18.1.0" + - "v18.1", "v18.1.0" + - "18.1-trixie", "18.1-bookworm" + - "18", "v18" + + Args: + tag_name: The tag name to parse + + Returns: + tuple: A tuple of integers representing (major, minor, patch, ...), or None if not a version tag + """ + # Handle common prefixes + normalized = tag_name.lower() + if normalized.startswith('v') and len(normalized) > 1 and normalized[1].isdigit(): + normalized = normalized[1:] + + # Skip non-version tags + if not any(c.isdigit() for c in normalized): + return None + + # Skip tags that don't start with a digit + if not normalized[0].isdigit(): + return None + + # Extract the version part (before any '-', '_', or non-numeric suffix) + version_part = normalized.split('-')[0].split('_')[0] + + # Split by '.' and try to parse as integers + try: + version_numbers = [] + for part in version_part.split('.'): + # Only take numeric parts + if part.isdigit(): + version_numbers.append(int(part)) + else: + # If we hit a non-numeric part, stop parsing + break + + if version_numbers: + return tuple(version_numbers) + else: + return None + except (ValueError, AttributeError): + return None + + def _parse_datetime(self, datetime_str: str | None) -> datetime: + """ + Parse a datetime string to a datetime object. + + Handles ISO format with 'Z' timezone suffix. + + Args: + datetime_str: The datetime string to parse (ISO format or None) + + Returns: + datetime: Parsed datetime object, or datetime.min if parsing fails + """ + if not datetime_str: + return datetime.min + try: + return datetime.fromisoformat(datetime_str.replace('Z', '+00:00')) + except (ValueError, AttributeError): + return datetime.min + + def _extract_arch_digest(self, tag: dict[str, Any], architecture: str) -> str | None: + """ + Extract the digest for a specific architecture from a tag's images. + + Args: + tag: Tag dictionary containing 'images' list + architecture: Architecture string in format 'os/architecture' (e.g., 'linux/amd64') + + Returns: + str: The digest for the specified architecture, or None if not found + """ + arch_os, arch_variant = self._parse_arch(architecture) + for image in tag.get('images', []): + if image.get('architecture') == arch_variant and image.get('os') == arch_os: + return image.get('digest') + return None + + def _create_tag_data_dict(self, tag: dict[str, Any], architecture: str) -> dict[str, Any]: + """ + Create a standardized tag data dictionary from a Docker Hub tag. + + Args: + tag: Raw tag dictionary from Docker Hub API + architecture: Architecture string to extract digest for + + Returns: + dict: Standardized tag data dictionary with name, last_updated, size, and digest + """ + return { + 'name': tag['name'], + 'last_updated': tag.get('last_updated'), + 'size': tag.get('full_size', 0), + 'digest': self._extract_arch_digest(tag, architecture) + } + + def _version_sort_key(self, tag: dict[str, Any]) -> tuple: + """ + Return sort key for version-aware sorting. + + Priority (highest to lowest): + 1. 'latest' tag (always first) + 2. Tags with version numbers (sorted by version descending) + 3. Non-version tags (sorted by last_updated) + + Args: + tag: Tag dictionary with 'name' and 'last_updated' fields + + Returns: + tuple: A sort key that can be used with sorted() or list.sort() + """ + tag_name = tag['name'].lower() + + # Priority 1: 'latest' tag + if tag_name == 'latest': + return (2, (999, 999, 999, 999), datetime.max) + + # Priority 2: Version tags + version = self._parse_version(tag['name']) + if version: + # Pad version tuple to 4 elements for consistent comparison + padded_version = version + (0,) * (4 - len(version)) + updated = self._parse_datetime(tag.get('last_updated')) + return (1, padded_version[:4], updated) + + # Priority 3: Non-version tags + updated = self._parse_datetime(tag.get('last_updated')) + return (0, (0, 0, 0, 0), updated) + + def _sort_tags(self, tag_data: list[dict[str, Any]], sort_by: str) -> None: + """Sort tag data in place based on the sort_by parameter. + + Args: + tag_data: List of tag dictionaries to sort + sort_by: Sort method - 'version', 'updated', or 'default' (no sorting) + """ + if sort_by == "version": + # Version-aware sorting + tag_data.sort(key=self._version_sort_key, reverse=True) + elif sort_by == "updated": + # Sort by last_updated timestamp + tag_data.sort(key=lambda x: self._parse_datetime(x.get('last_updated')), reverse=True) + # else: sort_by == "default", keep original order + + def get_docker_hub_tags(self, image_name: str, limit: int = 10, architecture: str = "linux/amd64", sort_by: str = "version") -> list[dict[str, Any]]: + """Query Docker Hub for image tags with timestamp information. + + Args: + image_name: Name of the image to query + limit: Maximum number of tags to return (unused in fetching, used by caller) + architecture: Architecture to filter by (e.g., 'linux/amd64') + sort_by: Sort method - 'version' (default), 'updated', or 'default' (Docker Hub order) + + Returns: + list: List of tag dictionaries sorted according to sort_by parameter + """ # Parse repository name if '/' in image_name: namespace, repo = image_name.split('/', 1) @@ -52,20 +214,7 @@ def get_docker_hub_tags(self, image_name: str, limit: int = 10, architecture: st tag_data: list[dict[str, Any]] = [] for tag in data.get('results', []): - # Find the image info for the requested architecture - arch_digest = None - arch_os, arch_variant = self._parse_arch(architecture) - for image in tag.get('images', []): - if image.get('architecture') == arch_variant and image.get('os') == arch_os: - arch_digest = image.get('digest') - break - - tag_data.append({ - 'name': tag['name'], - 'last_updated': tag.get('last_updated'), - 'size': tag.get('full_size', 0), - 'digest': arch_digest - }) + tag_data.append(self._create_tag_data_dict(tag, architecture)) # Handle pagination if there are more tags while 'next' in data and data['next'] and len(tag_data) < 1000: # Limit to avoid too many requests @@ -74,37 +223,29 @@ def get_docker_hub_tags(self, image_name: str, limit: int = 10, architecture: st data = response.json() for tag in data.get('results', []): - # Find the image info for the requested architecture - arch_digest = None - arch_os, arch_variant = self._parse_arch(architecture) - for image in tag.get('images', []): - if image.get('architecture') == arch_variant and image.get('os') == arch_os: - arch_digest = image.get('digest') - break - - tag_data.append({ - 'name': tag['name'], - 'last_updated': tag.get('last_updated'), - 'size': tag.get('full_size', 0), - 'digest': arch_digest - }) + tag_data.append(self._create_tag_data_dict(tag, architecture)) - # Sort by last_updated in descending order (newest first) - def _iso(dt_str): - try: - return datetime.fromisoformat(dt_str.replace('Z', '+00:00')) - except ValueError: - return datetime.min - - tag_data.sort(key=lambda x: _iso(x['last_updated']) if x['last_updated'] else datetime.min, reverse=True) + # Sort based on sort_by parameter + self._sort_tags(tag_data, sort_by) except requests.exceptions.RequestException as e: print(f"Error querying Docker Hub: {e}", file=sys.stderr) return [] else: return tag_data - def get_registry_tags(self, registry_url: str, image_name: str, limit: int = 10, architecture: str = "linux/amd64") -> list[dict[str, Any]]: - """Query a registry API v2 for image tags and attempt to get creation time.""" + def get_registry_tags(self, registry_url: str, image_name: str, limit: int = 10, architecture: str = "linux/amd64", sort_by: str = "version") -> list[dict[str, Any]]: + """Query a registry API v2 for image tags and attempt to get creation time. + + Args: + registry_url: URL of the registry + image_name: Name of the image to query + limit: Maximum number of tags to return (unused in fetching, used by caller) + architecture: Architecture to filter by (e.g., 'linux/amd64') + sort_by: Sort method - 'version' (default), 'updated', or 'default' (registry order) + + Returns: + list: List of tag dictionaries sorted according to sort_by parameter + """ url: str = f"{registry_url}/v2/{image_name}/tags/list" try: response = requests.get(url, timeout=30) @@ -154,14 +295,8 @@ def get_registry_tags(self, registry_url: str, image_name: str, limit: int = 10, 'digest': None }) - # Sort by last_updated in descending order if available - def _httpdate(dt_str): - try: - return parsedate_to_datetime(dt_str) - except Exception: - return datetime.min - - tag_data.sort(key=lambda x: _httpdate(x['last_updated']) if x['last_updated'] else datetime.min, reverse=True) + # Sort based on sort_by parameter + self._sort_tags(tag_data, sort_by) except requests.exceptions.RequestException as e: print(f"Error querying registry: {e}", file=sys.stderr) return [] @@ -303,6 +438,7 @@ def get_image_tags(self, args: argparse.Namespace, limit: int | None = None) -> """Get tags for the image based on the provided arguments.""" _, should_output = self._get_output_flags(args) fetch_limit = limit if limit is not None else args.limit + sort_by = getattr(args, 'sort', 'version') # Default to 'version' if not specified registry_url, image_name, is_docker_hub = self._parse_image_reference(args.image, args.registry) @@ -310,11 +446,11 @@ def get_image_tags(self, args: argparse.Namespace, limit: int | None = None) -> if not is_docker_hub: if should_output: print(f"Querying registry {registry_url} for {image_name} (architecture: {args.architecture})...") - tags = self.get_registry_tags(registry_url, image_name, fetch_limit, args.architecture) + tags = self.get_registry_tags(registry_url, image_name, fetch_limit, args.architecture, sort_by) else: if should_output: print(f"Querying Docker Hub for {args.image} (architecture: {args.architecture})...") - tags = self.get_docker_hub_tags(args.image, fetch_limit, args.architecture) + tags = self.get_docker_hub_tags(args.image, fetch_limit, args.architecture, sort_by) return tags, registry_url, image_name, is_docker_hub @@ -467,6 +603,8 @@ def main() -> None: recent_parser = subparsers.add_parser('list-recent', help='List recent tags for an image') recent_parser.add_argument('image', help='Image name (e.g., nginx or registry.example.com/nginx)') recent_parser.add_argument('--limit', type=int, default=10, help='Maximum number of tags to display (default: 10)') + recent_parser.add_argument('--sort', choices=['version', 'updated', 'default'], default='version', + help='Sort order: version (by version number, default), updated (by last_updated timestamp), default (registry order)') # Create the parser for the "list-same-hash" command hash_parser = subparsers.add_parser('list-same-hash', help='List tags with the same hash')