From 3dfe15e8326587fe9784c8c84b497ea74f7bbc22 Mon Sep 17 00:00:00 2001 From: Jonah Lawrence Date: Tue, 3 Jan 2023 11:43:00 -0700 Subject: [PATCH 1/3] fix: Prevent reusing tokens when rate limited --- src/stats.php | 76 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 15 deletions(-) diff --git a/src/stats.php b/src/stats.php index ba54c068..e9b83055 100644 --- a/src/stats.php +++ b/src/stats.php @@ -43,12 +43,14 @@ function getContributionGraphs(string $user): array // Get the years the user has contributed $contributionYears = getContributionYears($user); // build a list of individual requests + $tokens = []; $requests = []; foreach ($contributionYears as $year) { // create query for year $query = buildContributionGraphQuery($user, $year); // create curl request - $requests[$year] = getGraphQLCurlHandle($query); + $tokens[$year] = getGitHubToken(); + $requests[$year] = getGraphQLCurlHandle($query, $tokens[$year]); } // build multi-curl handle $multi = curl_multi_init(); @@ -67,19 +69,22 @@ function getContributionGraphs(string $user): array $decoded = is_string($contents) ? json_decode($contents) : null; // if response is empty or invalid, retry request one time if (empty($decoded) || empty($decoded->data)) { - // if rate limit is exceeded, don't retry + // if rate limit is exceeded, don't retry with same token $message = $decoded->errors[0]->message ?? ($decoded->message ?? "An API error occurred."); if (str_contains($message, "rate limit exceeded")) { - error_log("Error: $message"); - continue; + removeGitHubToken($tokens[$year]); } $query = buildContributionGraphQuery($user, $year); - $request = getGraphQLCurlHandle($query); + $token = getGitHubToken(); + $request = getGraphQLCurlHandle($query, $token); $contents = curl_exec($request); $decoded = is_string($contents) ? json_decode($contents) : null; // if the response is still empty or invalid, log an error and skip the year if (empty($decoded) || empty($decoded->data)) { $message = $decoded->errors[0]->message ?? ($decoded->message ?? "An API error occurred."); + if (str_contains($message, "rate limit exceeded")) { + removeGitHubToken($token); + } error_log("Failed to decode response for $user's $year contributions after 2 attempts. $message"); continue; } @@ -118,16 +123,46 @@ function getGitHubTokens() return $tokens; } +/** + * Get a token from the token pool + * + * @throws AssertionError if no tokens are available + */ +function getGitHubToken() +{ + $all_tokens = getGitHubTokens(); + return $all_tokens[array_rand($all_tokens)]; +} + +/** + * Remove a token from the token pool + * + * @param string $token Token to remove + */ +function removeGitHubToken(string $token) +{ + $index = array_search($token, $GLOBALS["ALL_TOKENS"]); + if ($index !== false) { + unset($GLOBALS["ALL_TOKENS"][$index]); + } + // if there is no available token, throw an error + if (empty($GLOBALS["ALL_TOKENS"])) { + throw new AssertionError( + "We are being rate-limited! Check git.io/streak-ratelimit for details.", + 429 + ); + } +} + /** Create a CurlHandle for a POST request to GitHub's GraphQL API * * @param string $query GraphQL query + * @param string $token GitHub token to use for the request * * @return CurlHandle The curl handle for the request */ -function getGraphQLCurlHandle(string $query) +function getGraphQLCurlHandle(string $query, string $token) { - $all_tokens = getGitHubTokens(); - $token = $all_tokens[array_rand($all_tokens)]; $headers = [ "Authorization: bearer $token", "Content-Type: application/json", @@ -158,12 +193,17 @@ function getGraphQLCurlHandle(string $query) */ function fetchGraphQL(string $query): stdClass { - $ch = getGraphQLCurlHandle($query); + $token = getGitHubToken(); + $ch = getGraphQLCurlHandle($query, $token); $response = curl_exec($ch); curl_close($ch); - $obj = is_string($response) ? json_decode($response) : null; + $decoded = is_string($response) ? json_decode($response) : null; // handle curl errors - if ($response === false || $obj === null || curl_getinfo($ch, CURLINFO_HTTP_CODE) >= 400) { + if ($response === false || $decoded === null || curl_getinfo($ch, CURLINFO_HTTP_CODE) >= 400) { + $message = $decoded->errors[0]->message ?? ($decoded->message ?? "An API error occurred."); + if (str_contains($message, "rate limit exceeded")) { + removeGitHubToken($token); + } // set response code to curl error code http_response_code(curl_getinfo($ch, CURLINFO_HTTP_CODE)); // Missing SSL certificate @@ -171,8 +211,8 @@ function fetchGraphQL(string $query): stdClass throw new AssertionError("You don't have a valid SSL Certificate installed or XAMPP.", 400); } // Handle errors such as "Bad credentials" - if ($obj && $obj->message) { - throw new AssertionError("Error: $obj->message \n", 401); + if ($message !== "An API error occurred.") { + throw new AssertionError("Error: $message \n", 401); } // Handle curl errors if (curl_errno($ch)) { @@ -180,7 +220,7 @@ function fetchGraphQL(string $query): stdClass } throw new AssertionError("An error occurred when getting a response from GitHub.\n", 502); } - return $obj; + return $decoded; } /** @@ -201,7 +241,13 @@ function getContributionYears(string $user): array } } }"; - $response = fetchGraphQL($query); + try { + $response = fetchGraphQL($query); + } catch (AssertionError $e) { + // retry once if an error occurred + error_log("An error occurred getting contribution years for $user: " . $e->getMessage()); + $response = fetchGraphQL($query); + } // User not found if (!empty($response->errors)) { $type = $response->errors[0]->type ?? ""; From 827e897f2110d5a78aeddeebca141dbd7d59ae06 Mon Sep 17 00:00:00 2001 From: Jonah Lawrence Date: Tue, 3 Jan 2023 11:54:14 -0700 Subject: [PATCH 2/3] refactor --- src/stats.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/stats.php b/src/stats.php index e9b83055..4539ac2c 100644 --- a/src/stats.php +++ b/src/stats.php @@ -186,14 +186,14 @@ function getGraphQLCurlHandle(string $query, string $token) * Create a POST request to GitHub's GraphQL API * * @param string $query GraphQL query + * @param string $token GitHub token to use for the request * * @return stdClass An object from the json response of the request * * @throws AssertionError If SSL verification fails */ -function fetchGraphQL(string $query): stdClass +function fetchGraphQL(string $query, string $token): stdClass { - $token = getGitHubToken(); $ch = getGraphQLCurlHandle($query, $token); $response = curl_exec($ch); curl_close($ch); @@ -242,11 +242,11 @@ function getContributionYears(string $user): array } }"; try { - $response = fetchGraphQL($query); + $response = fetchGraphQL($query, getGitHubToken()); } catch (AssertionError $e) { // retry once if an error occurred error_log("An error occurred getting contribution years for $user: " . $e->getMessage()); - $response = fetchGraphQL($query); + $response = fetchGraphQL($query, getGitHubToken()); } // User not found if (!empty($response->errors)) { From 1cd86273c9ed6eb97c74f72a923cc982c4a07be0 Mon Sep 17 00:00:00 2001 From: Jonah Lawrence Date: Tue, 3 Jan 2023 11:57:56 -0700 Subject: [PATCH 3/3] refactor --- src/stats.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/stats.php b/src/stats.php index 4539ac2c..c81fe1f7 100644 --- a/src/stats.php +++ b/src/stats.php @@ -74,6 +74,7 @@ function getContributionGraphs(string $user): array if (str_contains($message, "rate limit exceeded")) { removeGitHubToken($tokens[$year]); } + error_log("First attempt to decode response for $user's $year contributions failed. $message"); $query = buildContributionGraphQuery($user, $year); $token = getGitHubToken(); $request = getGraphQLCurlHandle($query, $token); @@ -200,7 +201,7 @@ function fetchGraphQL(string $query, string $token): stdClass $decoded = is_string($response) ? json_decode($response) : null; // handle curl errors if ($response === false || $decoded === null || curl_getinfo($ch, CURLINFO_HTTP_CODE) >= 400) { - $message = $decoded->errors[0]->message ?? ($decoded->message ?? "An API error occurred."); + $message = $decoded->errors[0]->message ?? ($decoded->message ?? ""); if (str_contains($message, "rate limit exceeded")) { removeGitHubToken($token); } @@ -211,7 +212,7 @@ function fetchGraphQL(string $query, string $token): stdClass throw new AssertionError("You don't have a valid SSL Certificate installed or XAMPP.", 400); } // Handle errors such as "Bad credentials" - if ($message !== "An API error occurred.") { + if ($message) { throw new AssertionError("Error: $message \n", 401); } // Handle curl errors