diff --git a/.gitignore b/.gitignore index b87fae6..40a2e17 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,9 @@ __pycache__/ venv .eggs .pytest_cache -*.egg-info \ No newline at end of file +*.egg-info +.vscode +test-env +github-to-sqlite.py +test.db +auth.json \ No newline at end of file diff --git a/github_to_sqlite/cli.py b/github_to_sqlite/cli.py index 98b99a5..8b32812 100644 --- a/github_to_sqlite/cli.py +++ b/github_to_sqlite/cli.py @@ -54,7 +54,11 @@ def auth(auth): type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True), help="Load issues JSON from this file instead of the API", ) -def issues(db_path, repo, issue, auth, load): +@click.option( + "--rev_req", + is_flag=True +) +def issues(db_path, repo, issue, auth, load, rev_req): "Save issues for a specified repository, e.g. simonw/datasette" db = sqlite_utils.Database(db_path) token = load_token(auth) @@ -65,7 +69,14 @@ def issues(db_path, repo, issue, auth, load): issues = list(issues) utils.save_issues(db, issues) - + + # Get PR reivews + if rev_req: + pulls = utils.is_open_pr(issues) + for pull in pulls: + reviews = utils.fetch_reviews(repo, token, pull) + reviews = list(reviews) + utils.save_reviews(db, reviews) @cli.command(name="issue-comments") @click.argument( diff --git a/github_to_sqlite/utils.py b/github_to_sqlite/utils.py index 3c51d36..7113119 100644 --- a/github_to_sqlite/utils.py +++ b/github_to_sqlite/utils.py @@ -46,6 +46,39 @@ def save_issues(db, issues): for label in labels: table.m2m("labels", label, pk="id") +def save_reviews(db, reviews): + if "reviews" not in db.table_names(): + db["reviews"].create({"id": int}, pk="id") + for original in reviews: + # Ignore all of the _url fields + review = { + key: value for key, value in original.items() if not (key.endswith("url") or key.endswith("_links")) + } + pr_url = original["pull_request_url"].split("/") + # Add repo key + review["repo"] = "{}/{}".format(pr_url[-4], pr_url[-3]) + # Add pull key + pr_num = pr_url[-1] + issue_rows = list( + db["issues"].rows_where( + "number = :number and repo = :repo", + {"repo": review["repo"], "number": pr_num}, + ) + ) + if len(issue_rows) == 1: + review["pull_request"] = issue_rows[0]["id"] + # Extract user + review["user"] = save_user(db, review["user"]) + # Insert Record + table = db["reviews"].upsert( + review, + pk="id", + foreign_keys=[ + ("user", "users", "id"), + ("pull_request", "issues", "id"), + ], + alter=True, + ) def save_user(db, user): # Remove all url fields except avatar_url and html_url @@ -204,8 +237,13 @@ def fetch_all_repos(username=None, token=None): for repos in paginate(url, headers): yield from repos +def fetch_reviews(repo, token=None, pull=None): + headers = make_headers(token) + url = "https://api.github.com/repos/{}/pulls/{}/reviews".format(repo, pull["number"]) + for reviews in paginate(url, headers): + yield from reviews -def fetch_user(username=None, token=None): +def fetch_user(repo=None, token=None): assert username or token, "Must provide username= or token= or both" headers = make_headers(token) if username: @@ -214,6 +252,12 @@ def fetch_user(username=None, token=None): url = "https://api.github.com/user" return requests.get(url, headers=headers).json() +def is_open_pr(issues): + pulls = list() + for issue in issues: + if issue.get("state")=="open" and issue.get("pull_request"): + pulls.append(issue) + return pulls def paginate(url, headers=None): while url: diff --git a/setup.py b/setup.py index e970c88..08f8e19 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup import os -VERSION = "0.6" +VERSION = "0.7" def get_long_description():