From fefd96f2c8b88e6b1118b68b95cc53e7e9cd087a Mon Sep 17 00:00:00 2001 From: jelleas Date: Thu, 14 Aug 2025 15:42:26 +0200 Subject: [PATCH] gh releases => commits on default branch --- README.md | 7 ++- checkpy/__main__.py | 2 +- checkpy/database/database.py | 30 ++++++---- checkpy/downloader/downloader.py | 96 ++++++++++++++++++++------------ setup.py | 2 +- 5 files changed, 86 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index fefa5e9..eeca4b6 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,9 @@ Here spcourse/tests points to . You can also --dev get extra information to support the development of tests --silent do not print test results to stdout --json return output as json, implies silent - --gh-auth GH_AUTH username:personal_access_token for authentication with GitHub. Only used to increase GitHub api's rate limit. + --gh-auth GH_AUTH username:personal_access_token for authentication with GitHub. + --output-limit OUTPUTLIMIT + limit the number of characters stored for each test's output field. Default is 1000. Set to 0 to disable this limit. To test a single file call: @@ -233,8 +235,7 @@ correctForPos = test()(declarative ### Distributing tests -checkpy downloads tests directly from Github repos. The requirement is that a folder called ``tests`` exists within the repo that contains only tests and folders (which checkpy treats as modules). There must also be at least one release in the Github repo. checkpy will automatically target the latest release. To download tests call checkpy with the optional ``-d`` argument and pass your github repo url. checkpy will automatically keep tests up to date by checking for any new releases on GitHub. - +checkpy downloads tests directly from Github repos. The requirement is that a folder called ``tests`` exists within the repo that contains only tests and folders (which checkpy treats as modules). checkpy will pull from the default branch. To download tests call checkpy with the optional ``-d`` argument and pass your github repo url. checkpy will automatically keep tests up to date by checking for any new commits on GitHub. ### Testing checkpy diff --git a/checkpy/__main__.py b/checkpy/__main__.py index b2cfaab..eca9567 100644 --- a/checkpy/__main__.py +++ b/checkpy/__main__.py @@ -32,7 +32,7 @@ def main(): parser.add_argument("--dev", action="store_true", help="get extra information to support the development of tests") parser.add_argument("--silent", action="store_true", help="do not print test results to stdout") parser.add_argument("--json", action="store_true", help="return output as json, implies silent") - parser.add_argument("--gh-auth", action="store", help="username:personal_access_token for authentication with GitHub. Only used to increase GitHub api's rate limit.") + parser.add_argument("--gh-auth", action="store", help="username:personal_access_token for authentication with GitHub.") parser.add_argument("--output-limit", action="store", type=int, default=1000, dest="outputLimit", help="limit the number of characters stored for each test's output field. Default is 1000. Set to 0 to disable this limit.") parser.add_argument("files", action="store", nargs="*", help="names of files to be tested") args = parser.parse_args() diff --git a/checkpy/database/database.py b/checkpy/database/database.py index 38555b5..fece0c5 100644 --- a/checkpy/database/database.py +++ b/checkpy/database/database.py @@ -57,7 +57,12 @@ def isKnownGithub(username: str, repoName: str) -> bool: with githubTable() as table: return table.contains((query.user == username) & (query.repo == repoName)) -def addToGithubTable(username: str, repoName: str, releaseId: str, releaseTag: str): +def addToGithubTable( + username: str, + repoName: str, + commitMessage: str, + commitSha: str + ): if not isKnownGithub(username, repoName): path = str(checkpy.CHECKPYPATH / "tests" / repoName) @@ -66,8 +71,8 @@ def addToGithubTable(username: str, repoName: str, releaseId: str, releaseTag: s "user" : username, "repo" : repoName, "path" : path, - "release" : releaseId, - "tag" : releaseTag, + "message" : commitMessage, + "sha" : commitSha, "timestamp" : time.time() }) @@ -79,7 +84,12 @@ def addToLocalTable(localPath: pathlib.Path): "path" : str(localPath) }) -def updateGithubTable(username: str, repoName: str, releaseId: str, releaseTag: str): +def updateGithubTable( + username: str, + repoName: str, + commitMessage: str, + commitSha: str + ): query = tinydb.Query() path = str(checkpy.CHECKPYPATH / "tests" / repoName) with githubTable() as table: @@ -87,8 +97,8 @@ def updateGithubTable(username: str, repoName: str, releaseId: str, releaseTag: "user" : username, "repo" : repoName, "path" : path, - "release" : releaseId, - "tag" : releaseTag, + "message" : commitMessage, + "sha" : commitSha, "timestamp" : time.time() }, query.user == username and query.repo == repoName) @@ -110,12 +120,12 @@ def githubPath(username: str, repoName: str) -> pathlib.Path: with githubTable() as table: return pathlib.Path(table.search(query.user == username and query.repo == repoName)[0]["path"]) -def releaseId(username: str, repoName: str) -> str: +def commitSha(username: str, repoName: str) -> str: query = tinydb.Query() with githubTable() as table: - return table.search(query.user == username and query.repo == repoName)[0]["release"] + return table.search(query.user == username and query.repo == repoName)[0]["sha"] -def releaseTag(username: str, repoName: str) -> str: +def commitMessage(username: str, repoName: str) -> str: query = tinydb.Query() with githubTable() as table: - return table.search(query.user == username and query.repo == repoName)[0]["tag"] + return table.search(query.user == username and query.repo == repoName)[0]["message"] diff --git a/checkpy/downloader/downloader.py b/checkpy/downloader/downloader.py index cdeb3de..fff296f 100644 --- a/checkpy/downloader/downloader.py +++ b/checkpy/downloader/downloader.py @@ -32,7 +32,7 @@ def download(githubLink: str): repoName = githubLink.split("/")[-1].lower() try: - _syncRelease(username, repoName) + _syncCommit(username, repoName) _download(username, repoName) except exception.DownloadError as e: printer.displayError(str(e)) @@ -49,7 +49,7 @@ def register(localLink: Union[str, pathlib.Path]): def update(): for username, repoName in database.forEachUserAndRepo(): try: - _syncRelease(username, repoName) + _syncCommit(username, repoName) _download(username, repoName) except exception.DownloadError as e: printer.displayError(str(e)) @@ -75,79 +75,103 @@ def updateSilently(): database.setTimestampGithub(username, repoName) try: - if _newReleaseAvailable(username, repoName): + if _newCommitAvailable(username, repoName): _download(username, repoName) - except exception.DownloadError as e: + except exception.DownloadError: pass -def _newReleaseAvailable(githubUserName: str, githubRepoName: str) -> bool: +def _newCommitAvailable(githubUserName: str, githubRepoName: str) -> bool: # unknown/new download if not database.isKnownGithub(githubUserName, githubRepoName): return True - releaseJson = _getReleaseJson(githubUserName, githubRepoName) - - # new release id found - if releaseJson["id"] != database.releaseId(githubUserName, githubRepoName): - database.updateGithubTable(githubUserName, githubRepoName, releaseJson["id"], releaseJson["tag_name"]) + commitJson = _getLatestCommitJson(githubUserName, githubRepoName) + + # new commit found + if commitJson["sha"] != database.commitSha(githubUserName, githubRepoName): + database.updateGithubTable( + githubUserName, + githubRepoName, + commitJson["commit"]["message"], + commitJson["sha"], + ) return True - # no new release found + # no new commit found return False -def _syncRelease(githubUserName: str, githubRepoName: str): - releaseJson = _getReleaseJson(githubUserName, githubRepoName) +def _syncCommit(githubUserName: str, githubRepoName: str): + commitJson = _getLatestCommitJson(githubUserName, githubRepoName) if database.isKnownGithub(githubUserName, githubRepoName): - database.updateGithubTable(githubUserName, githubRepoName, releaseJson["id"], releaseJson["tag_name"]) + database.updateGithubTable( + githubUserName, + githubRepoName, + commitJson["commit"]["message"], + commitJson["sha"], + ) else: - database.addToGithubTable(githubUserName, githubRepoName, releaseJson["id"], releaseJson["tag_name"]) - + database.addToGithubTable( + githubUserName, + githubRepoName, + commitJson["commit"]["message"], + commitJson["sha"], + ) + +def _get_with_auth(url: str) -> requests.Response: + """ + Get a url with authentication if available. + Returns a requests.Response object. + """ + global user + global personal_access_token + if user and personal_access_token: + return requests.get(url, auth=(user, personal_access_token)) + else: + return requests.get(url) -def _getReleaseJson(githubUserName: str, githubRepoName: str) -> Dict: +def _getLatestCommitJson(githubUserName: str, githubRepoName: str) -> Dict: """ + Get the latest commit from the default branch of the given repository. This performs one api call, beware of rate limit!!! Returns a dictionary representing the json returned by github In case of an error, raises an exception.DownloadError """ - apiReleaseLink = f"https://api.github.com/repos/{githubUserName}/{githubRepoName}/releases/latest" + apiCommitLink = f"https://api.github.com/repos/{githubUserName}/{githubRepoName}/commits" - global user - global personal_access_token try: - if user and personal_access_token: - r = requests.get(apiReleaseLink, auth=(user, personal_access_token)) - else: - r = requests.get(apiReleaseLink) + r = _get_with_auth(apiCommitLink) except requests.exceptions.ConnectionError as e: raise exception.DownloadError(message="Oh no! It seems like there is no internet connection available?!") # exceeded rate limit, if r.status_code == 403: - raise exception.DownloadError(message=f"Tried finding new releases from {githubUserName}/{githubRepoName} but exceeded the rate limit, try again within an hour!") + raise exception.DownloadError(message=f"Tried finding new commits from {githubUserName}/{githubRepoName} but exceeded the rate limit, try again within an hour!") - # no releases found or page not found + # no commits found or page not found if r.status_code == 404: - raise exception.DownloadError(message=f"Failed to check for new tests from {githubUserName}/{githubRepoName} because: no releases found (404)") + raise exception.DownloadError(message=f"Failed to check for new commits from {githubUserName}/{githubRepoName} because: no commits found (404)") # random error if not r.ok: - raise exception.DownloadError(message=f"Failed to sync releases from {githubUserName}/{githubRepoName} because: {r.reason}") + raise exception.DownloadError(message=f"Failed to get commits from {githubUserName}/{githubRepoName} because: {r.reason}") - return r.json() + return r.json()[0] -# download tests for githubUserName and githubRepoName from what is known in downloadlocations.json -# use _syncRelease() to force an update in downloadLocations.json +# download tests for githubUserName and githubRepoName from what is known in db +# use _syncCommit() to force an update in db def _download(githubUserName: str, githubRepoName: str): - githubLink = f"https://github.com/{githubUserName}/{githubRepoName}" - zipLink = githubLink + f"/archive/{database.releaseTag(githubUserName, githubRepoName)}.zip" + sha = database.commitSha(githubUserName, githubRepoName) + zipUrl = f'https://api.github.com/repos/{githubUserName}/{githubRepoName}/zipball/{sha}' try: - r = requests.get(zipLink) + r = _get_with_auth(zipUrl) except requests.exceptions.ConnectionError as e: raise exception.DownloadError(message = "Oh no! It seems like there is no internet connection available?!") + gitHubUrl = f'https://github.com/{githubUserName}/{githubRepoName}' # just for feedback + if not r.ok: - raise exception.DownloadError(message = f"Failed to download {githubLink} because: {r.reason}") + raise exception.DownloadError(message = f"Failed to download {gitHubUrl} because: {r.reason}") f = io.BytesIO(r.content) @@ -177,7 +201,7 @@ def _download(githubUserName: str, githubRepoName: str): _extractTests(z, destPath) - printer.displayCustom(f"Finished downloading: {githubLink}") + printer.displayCustom(f"Finished downloading: {gitHubUrl}") def _extractTests(zipfile: zf.ZipFile, destPath: pathlib.Path): if not destPath.exists(): diff --git a/setup.py b/setup.py index 77ba983..ed09cc5 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='checkPy', - version='2.1.0', + version='2.1.1', description='A simple python testing framework for educational purposes', long_description=long_description,