gh releases => commits on default branch

README.md

@@ -53,7 +53,9 @@ Here spcourse/tests points to <https://github.com/spcourse/tests>. You can also
     --dev                 get extra information to support the development of tests
     --silent              do not print test results to stdout
     --json                return output as json, implies silent
-    --gh-auth GH_AUTH     username:personal_access_token for authentication with GitHub. Only used to increase  GitHub api's rate limit.
+    --gh-auth GH_AUTH     username:personal_access_token for authentication with GitHub.
+    --output-limit OUTPUTLIMIT
+                          limit the number of characters stored for each test's output field. Default is 1000. Set to 0 to disable this limit.
 
 To test a single file call:
 
@@ -233,8 +235,7 @@ correctForPos = test()(declarative
 
 ### Distributing tests
 
-checkpy downloads tests directly from Github repos. The requirement is that a folder called ``tests`` exists within the repo that contains only tests and folders (which checkpy treats as modules). There must also be at least one release in the Github repo. checkpy will automatically target the latest release. To download tests call checkpy with the optional ``-d`` argument and pass your github repo url. checkpy will automatically keep tests up to date by checking for any new releases on GitHub.
-
+checkpy downloads tests directly from Github repos. The requirement is that a folder called ``tests`` exists within the repo that contains only tests and folders (which checkpy treats as modules). checkpy will pull from the default branch. To download tests call checkpy with the optional ``-d`` argument and pass your github repo url. checkpy will automatically keep tests up to date by checking for any new commits on GitHub.
 
 ### Testing checkpy
 

checkpy/__main__.py

@@ -32,7 +32,7 @@ def main():
     parser.add_argument("--dev", action="store_true", help="get extra information to support the development of tests")
     parser.add_argument("--silent", action="store_true", help="do not print test results to stdout")
     parser.add_argument("--json", action="store_true", help="return output as json, implies silent")
-    parser.add_argument("--gh-auth", action="store", help="username:personal_access_token for authentication with GitHub. Only used to increase GitHub api's rate limit.")
+    parser.add_argument("--gh-auth", action="store", help="username:personal_access_token for authentication with GitHub.")
     parser.add_argument("--output-limit", action="store", type=int, default=1000, dest="outputLimit", help="limit the number of characters stored for each test's output field. Default is 1000. Set to 0 to disable this limit.")
     parser.add_argument("files", action="store", nargs="*", help="names of files to be tested")
     args = parser.parse_args()

checkpy/database/database.py

@@ -57,7 +57,12 @@ def isKnownGithub(username: str, repoName: str) -> bool:
     with githubTable() as table:
         return table.contains((query.user == username) & (query.repo == repoName))
 
-def addToGithubTable(username: str, repoName: str, releaseId: str, releaseTag: str):
+def addToGithubTable(
+        username: str,
+        repoName: str,
+        commitMessage: str,
+        commitSha: str
+    ):
     if not isKnownGithub(username, repoName):
         path = str(checkpy.CHECKPYPATH / "tests" / repoName)
 
@@ -66,8 +71,8 @@ def addToGithubTable(username: str, repoName: str, releaseId: str, releaseTag: s
                 "user" 			: username,
                 "repo" 			: repoName,
                 "path"          : path,
-                "release" 		: releaseId,
-                "tag" 			: releaseTag,
+                "message" 		: commitMessage,
+                "sha" 		    : commitSha,
                 "timestamp" 	: time.time()
             })
 
@@ -79,16 +84,21 @@ def addToLocalTable(localPath: pathlib.Path):
                 "path" : str(localPath)
             })
 
-def updateGithubTable(username: str, repoName: str, releaseId: str, releaseTag: str):
+def updateGithubTable(
+        username: str,
+        repoName: str,
+        commitMessage: str,
+        commitSha: str
+    ):
     query = tinydb.Query()
     path = str(checkpy.CHECKPYPATH / "tests" / repoName)
     with githubTable() as table:
         table.update({
             "user" 			: username,
             "repo" 			: repoName,
             "path"          : path,
-            "release" 		: releaseId,
-            "tag" 			: releaseTag,
+            "message" 		: commitMessage,
+            "sha" 		    : commitSha,
             "timestamp" 	: time.time()
         }, query.user == username and query.repo == repoName)
 
@@ -110,12 +120,12 @@ def githubPath(username: str, repoName: str) -> pathlib.Path:
     with githubTable() as table:
         return pathlib.Path(table.search(query.user == username and query.repo == repoName)[0]["path"])
 
-def releaseId(username: str, repoName: str) -> str:
+def commitSha(username: str, repoName: str) -> str:
     query = tinydb.Query()
     with githubTable() as table:
-        return table.search(query.user == username and query.repo == repoName)[0]["release"]
+        return table.search(query.user == username and query.repo == repoName)[0]["sha"]
 
-def releaseTag(username: str, repoName: str) -> str:
+def commitMessage(username: str, repoName: str) -> str:
     query = tinydb.Query()
     with githubTable() as table:
-        return table.search(query.user == username and query.repo == repoName)[0]["tag"]
+        return table.search(query.user == username and query.repo == repoName)[0]["message"]

checkpy/downloader/downloader.py

@@ -32,7 +32,7 @@ def download(githubLink: str):
     repoName = githubLink.split("/")[-1].lower()
 
     try:
-        _syncRelease(username, repoName)
+        _syncCommit(username, repoName)
         _download(username, repoName)
     except exception.DownloadError as e:
         printer.displayError(str(e))
@@ -49,7 +49,7 @@ def register(localLink: Union[str, pathlib.Path]):
 def update():
     for username, repoName in database.forEachUserAndRepo():
         try:
-            _syncRelease(username, repoName)
+            _syncCommit(username, repoName)
             _download(username, repoName)
         except exception.DownloadError as e:
             printer.displayError(str(e))
@@ -75,79 +75,103 @@ def updateSilently():
 
         database.setTimestampGithub(username, repoName)
         try:
-            if _newReleaseAvailable(username, repoName):
+            if _newCommitAvailable(username, repoName):
                 _download(username, repoName)
-        except exception.DownloadError as e:
+        except exception.DownloadError:
             pass
 
-def _newReleaseAvailable(githubUserName: str, githubRepoName: str) -> bool:
+def _newCommitAvailable(githubUserName: str, githubRepoName: str) -> bool:
     # unknown/new download
     if not database.isKnownGithub(githubUserName, githubRepoName):
         return True
-    releaseJson = _getReleaseJson(githubUserName, githubRepoName)
-
-    # new release id found
-    if releaseJson["id"] != database.releaseId(githubUserName, githubRepoName):
-        database.updateGithubTable(githubUserName, githubRepoName, releaseJson["id"], releaseJson["tag_name"])
+    commitJson = _getLatestCommitJson(githubUserName, githubRepoName)
+
+    # new commit found
+    if commitJson["sha"] != database.commitSha(githubUserName, githubRepoName):
+        database.updateGithubTable(
+            githubUserName,
+            githubRepoName,
+            commitJson["commit"]["message"],
+            commitJson["sha"],
+        )
         return True
 
-    # no new release found
+    # no new commit found
     return False
 
-def _syncRelease(githubUserName: str, githubRepoName: str):
-    releaseJson = _getReleaseJson(githubUserName, githubRepoName)
+def _syncCommit(githubUserName: str, githubRepoName: str):
+    commitJson = _getLatestCommitJson(githubUserName, githubRepoName)
 
     if database.isKnownGithub(githubUserName, githubRepoName):
-        database.updateGithubTable(githubUserName, githubRepoName, releaseJson["id"], releaseJson["tag_name"])
+        database.updateGithubTable(
+            githubUserName,
+            githubRepoName,
+            commitJson["commit"]["message"],
+            commitJson["sha"],
+        )
     else:
-        database.addToGithubTable(githubUserName, githubRepoName, releaseJson["id"], releaseJson["tag_name"])
-
+        database.addToGithubTable(
+            githubUserName,
+            githubRepoName,
+            commitJson["commit"]["message"],
+            commitJson["sha"],
+        )
+
+def _get_with_auth(url: str) -> requests.Response:
+    """
+    Get a url with authentication if available.
+    Returns a requests.Response object.
+    """
+    global user
+    global personal_access_token
+    if user and personal_access_token:
+        return requests.get(url, auth=(user, personal_access_token))
+    else:
+        return requests.get(url)
 
-def _getReleaseJson(githubUserName: str, githubRepoName: str) -> Dict:
+def _getLatestCommitJson(githubUserName: str, githubRepoName: str) -> Dict:
     """
+    Get the latest commit from the default branch of the given repository.
     This performs one api call, beware of rate limit!!!
     Returns a dictionary representing the json returned by github
     In case of an error, raises an exception.DownloadError
     """
-    apiReleaseLink = f"https://api.github.com/repos/{githubUserName}/{githubRepoName}/releases/latest"
+    apiCommitLink = f"https://api.github.com/repos/{githubUserName}/{githubRepoName}/commits"
 
-    global user
-    global personal_access_token
     try:
-        if user and personal_access_token:
-            r = requests.get(apiReleaseLink, auth=(user, personal_access_token))
-        else:
-            r = requests.get(apiReleaseLink)
+        r = _get_with_auth(apiCommitLink)
     except requests.exceptions.ConnectionError as e:
         raise exception.DownloadError(message="Oh no! It seems like there is no internet connection available?!")
 
     # exceeded rate limit,
     if r.status_code == 403:
-        raise exception.DownloadError(message=f"Tried finding new releases from {githubUserName}/{githubRepoName} but exceeded the rate limit, try again within an hour!")
+        raise exception.DownloadError(message=f"Tried finding new commits from {githubUserName}/{githubRepoName} but exceeded the rate limit, try again within an hour!")
 
-    # no releases found or page not found
+    # no commits found or page not found
     if r.status_code == 404:
-        raise exception.DownloadError(message=f"Failed to check for new tests from {githubUserName}/{githubRepoName} because: no releases found (404)")
+        raise exception.DownloadError(message=f"Failed to check for new commits from {githubUserName}/{githubRepoName} because: no commits found (404)")
 
     # random error
     if not r.ok:
-        raise exception.DownloadError(message=f"Failed to sync releases from {githubUserName}/{githubRepoName} because: {r.reason}")
+        raise exception.DownloadError(message=f"Failed to get commits from {githubUserName}/{githubRepoName} because: {r.reason}")
 
-    return r.json()
+    return r.json()[0]
 
-# download tests for githubUserName and githubRepoName from what is known in downloadlocations.json
-# use _syncRelease() to force an update in downloadLocations.json
+# download tests for githubUserName and githubRepoName from what is known in db
+# use _syncCommit() to force an update in db
 def _download(githubUserName: str, githubRepoName: str):
-    githubLink = f"https://github.com/{githubUserName}/{githubRepoName}"
-    zipLink = githubLink + f"/archive/{database.releaseTag(githubUserName, githubRepoName)}.zip"
+    sha = database.commitSha(githubUserName, githubRepoName)
+    zipUrl = f'https://api.github.com/repos/{githubUserName}/{githubRepoName}/zipball/{sha}'
 
     try:
-        r = requests.get(zipLink)
+        r = _get_with_auth(zipUrl)
     except requests.exceptions.ConnectionError as e:
         raise exception.DownloadError(message = "Oh no! It seems like there is no internet connection available?!")
 
+    gitHubUrl = f'https://github.com/{githubUserName}/{githubRepoName}' # just for feedback
+
     if not r.ok:
-        raise exception.DownloadError(message = f"Failed to download {githubLink} because: {r.reason}")
+        raise exception.DownloadError(message = f"Failed to download {gitHubUrl} because: {r.reason}")
 
     f = io.BytesIO(r.content)
 
@@ -177,7 +201,7 @@ def _download(githubUserName: str, githubRepoName: str):
 
         _extractTests(z, destPath)
 
-    printer.displayCustom(f"Finished downloading: {githubLink}")
+    printer.displayCustom(f"Finished downloading: {gitHubUrl}")
 
 def _extractTests(zipfile: zf.ZipFile, destPath: pathlib.Path):
     if not destPath.exists():

setup.py

@@ -11,7 +11,7 @@
 setup(
 	name='checkPy',
 
-	version='2.1.0',
+	version='2.1.1',
 
 	description='A simple python testing framework for educational purposes',
 	long_description=long_description,