diff --git a/github_to_sqlite/cli.py b/github_to_sqlite/cli.py index 70aa3b5..b181bf8 100644 --- a/github_to_sqlite/cli.py +++ b/github_to_sqlite/cli.py @@ -602,6 +602,39 @@ def workflows(db_path, repos, auth): utils.ensure_db_shape(db) +@cli.command() +@click.argument( + "db_path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.option( + "-a", + "--auth", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=True), + default="auth.json", + help="Path to auth.json token file", +) +@click.option( + "--load", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=True, exists=True), + help="Load gists JSON from this file instead of the API", +) +def gists(db_path, auth, load): + "Save Gists for authenticated user" + db = sqlite_utils.Database(db_path) + token = load_token(auth) + if token is None: + raise click.ClickException("User must be authenticated") + if load: + gists_full = json.load(open(load)) + else: + gists_full = utils.fetch_gists(token) + + utils.save_gists(db, gists_full) + utils.ensure_db_shape(db) + + def load_token(auth): try: token = json.load(open(auth))["github_personal_token"] diff --git a/github_to_sqlite/utils.py b/github_to_sqlite/utils.py index f934d76..e966b80 100644 --- a/github_to_sqlite/utils.py +++ b/github_to_sqlite/utils.py @@ -224,11 +224,11 @@ def save_pull_requests(db, pull_requests, repo): def save_user(db, user): - # Under some conditions, GitHub caches removed repositories with + # Under some conditions, GitHub caches removed repositories with # stars and ends up leaving dangling `None` user references. if user is None: return None - + # Remove all url fields except avatar_url and html_url to_save = { key: value @@ -884,3 +884,34 @@ def save_workflow(db, repo_id, filename, content): pk="id", foreign_keys=["job", "repo"], ) + + +# gists_full = utils.fetch_gists(token) + +# utils.save_gists(db, gists_full) +# utils.ensure_db_shape(db) + + +def fetch_gists(token): + headers = make_headers(token) + headers["accept"] = "application/vnd.github+json" + for gists in paginate("https://api.github.com/gists", headers): + yield from gists + + +def save_gists(db, gists): + # For each gist need to fetch the raw file content + gists_table = db.table("gists") + gist_files_table = db.table("gist_files") + for gist in gists: + id = gist["id"] + gists_table.insert(gist, pk="id", alter=True, replace=True) + for file in gist["files"].values(): + file["id"] = "{}:{}".format(id, file["filename"]) + file["gist"] = id + # Try to retrieve the content + response = requests.get(file["raw_url"]) + + gist_files_table.insert( + file, pk="id", alter=True, replace=True, foreign_keys=["gist"] + )