From 5c11751f9b79b44319b657b6e5e3946e6adcfe62 Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Wed, 6 Jul 2022 16:07:48 +0200 Subject: [PATCH 1/2] feat(Fetcher): fetch_git implementation --- .gitignore | 3 +++ getdeck/deckfile/file.py | 2 ++ getdeck/sources/file.py | 30 +++++++++++++++++++++++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b48a8ac..67dc125 100644 --- a/.gitignore +++ b/.gitignore @@ -341,3 +341,6 @@ ehthumbs.db Gemfile.lock beautiful-jekyll-theme-*.gem + +# vscode +.vscode \ No newline at end of file diff --git a/getdeck/deckfile/file.py b/getdeck/deckfile/file.py index 9089a6b..76688bb 100644 --- a/getdeck/deckfile/file.py +++ b/getdeck/deckfile/file.py @@ -52,6 +52,8 @@ class DeckfileFileSource(BaseModel): type: str = "file" ref: str = None content: Dict = None + targetRevision: str = "" + path: str = "" class DeckfileKustomizeSource(BaseModel): diff --git a/getdeck/sources/file.py b/getdeck/sources/file.py index c53848b..a4a514b 100644 --- a/getdeck/sources/file.py +++ b/getdeck/sources/file.py @@ -1,5 +1,7 @@ import logging from operator import methodcaller +import os +import tempfile from typing import List, Union import requests @@ -13,6 +15,7 @@ ) from getdeck.sources.types import K8sSourceFile from getdeck.utils import sniff_protocol +from git import Repo logger = logging.getLogger("deck") @@ -109,4 +112,29 @@ def fetch_local(self, **kwargs): raise e def fetch_git(self, **kwargs) -> List[K8sSourceFile]: - raise NotImplementedError + k8s_workload_files = [] + try: + with tempfile.TemporaryDirectory() as tmp_source: + logger.debug(f"Cloning from {self.source.ref} to {tmp_source}") + + if not self.source.path: + raise Exception("Path to file required.") + + repo = Repo.clone_from(self.source.ref, tmp_source) + if self.source.targetRevision: + repo.git.checkout(self.source.targetRevision) + + file_source = os.path.join(tmp_source, self.source.path) + with open(file_source, "r") as input_file: + docs = yaml.load_all(input_file.read(), Loader=yaml.FullLoader) + + for doc in docs: + if doc: + k8s_workload_files.append( + K8sSourceFile(name=self.source.ref, content=doc) + ) + + return k8s_workload_files + except Exception as e: + logger.error(f"Error loading files from git repository {e}") + raise e From fb64879138d216742258c4f314251f41fa1618cb Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Thu, 7 Jul 2022 09:35:21 +0200 Subject: [PATCH 2/2] feat(Fetcher): support fetching from directory the path of fetch_local and fetch_git can point to a folder and all yaml files within this folder are parsed --- getdeck/sources/file.py | 83 +++++++++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 23 deletions(-) diff --git a/getdeck/sources/file.py b/getdeck/sources/file.py index a4a514b..bf2af9a 100644 --- a/getdeck/sources/file.py +++ b/getdeck/sources/file.py @@ -20,6 +20,10 @@ logger = logging.getLogger("deck") +class FetcherError(Exception): + pass + + class Fetcher: def __init__( self, @@ -71,6 +75,51 @@ class FileFetcher(Fetcher): def not_supported_message(self): return f"Protocol {self.type} not supported for {type(self.source).__name__}" + @staticmethod + def _parse_source_file(ref: str) -> List[K8sSourceFile]: + with open(ref, "r") as input_file: + docs = yaml.load_all(input_file.read(), Loader=yaml.FullLoader) + + k8s_workload_files = [] + for doc in docs: + if doc: + k8s_workload_files.append(K8sSourceFile(name=ref, content=doc)) + return k8s_workload_files + + @staticmethod + def _parse_source_files(refs: List[str]) -> List[K8sSourceFile]: + k8s_workload_files = [] + for ref in refs: + workloads = FileFetcher._parse_source_file(ref=ref) + k8s_workload_files += workloads + return k8s_workload_files + + @staticmethod + def _parse_source_directory(ref: str) -> List[K8sSourceFile]: + refs = [] + + if not os.path.isdir(ref): + raise FetcherError( + f"The provided path does not point to a directory: {ref}" + ) + + extensions = (".yaml", ".yml") + for file in os.listdir(ref): + if file.endswith(extensions): + refs.append(os.path.join(ref, file)) + + # parse workloads + k8s_workload_files = FileFetcher._parse_source_files(refs=refs) + return k8s_workload_files + + @staticmethod + def _parse_source(ref: str) -> List[K8sSourceFile]: + if os.path.isdir(ref): + k8s_workload_files = FileFetcher._parse_source_directory(ref=ref) + else: + k8s_workload_files = FileFetcher._parse_source_file(ref=ref) + return k8s_workload_files + def fetch_content(self, **kwargs) -> List[K8sSourceFile]: return [K8sSourceFile(name="Deckfile", content=self.source.content)] @@ -89,52 +138,40 @@ def fetch_http(self, **kwargs) -> List[K8sSourceFile]: ) return k8s_workload_files except Exception as e: - logger.error(f"Error loading files from http {e}") + logger.error(f"Error loading file from http {e}") raise e def fetch_https(self, **kwargs): return self.fetch_http(**kwargs) def fetch_local(self, **kwargs): - k8s_workload_files = [] try: logger.debug(f"Reading file {self.source.ref}") - with open(self.source.ref, "r") as input_file: - docs = yaml.load_all(input_file.read(), Loader=yaml.FullLoader) - for doc in docs: - if doc: - k8s_workload_files.append( - K8sSourceFile(name=self.source.ref, content=doc) - ) + k8s_workload_files = self._parse_source(ref=self.source.ref) return k8s_workload_files except Exception as e: - logger.error(f"Error loading files from http {e}") + logger.error(f"Error loading file from http {e}") raise e def fetch_git(self, **kwargs) -> List[K8sSourceFile]: - k8s_workload_files = [] try: with tempfile.TemporaryDirectory() as tmp_source: logger.debug(f"Cloning from {self.source.ref} to {tmp_source}") - if not self.source.path: - raise Exception("Path to file required.") + source_path = "" + if self.source.path: + source_path = self.source.path + # clone & checkout repository repo = Repo.clone_from(self.source.ref, tmp_source) if self.source.targetRevision: repo.git.checkout(self.source.targetRevision) - file_source = os.path.join(tmp_source, self.source.path) - with open(file_source, "r") as input_file: - docs = yaml.load_all(input_file.read(), Loader=yaml.FullLoader) - - for doc in docs: - if doc: - k8s_workload_files.append( - K8sSourceFile(name=self.source.ref, content=doc) - ) + # file / directory + tmp_source_path = os.path.join(tmp_source, source_path) + k8s_workload_files = self._parse_source(ref=tmp_source_path) return k8s_workload_files except Exception as e: - logger.error(f"Error loading files from git repository {e}") + logger.error(f"Error loading file(s) from git repository {e}") raise e