Automatically remove definition owners with deleted GitHub accounts (D…

…efinitelyTyped#58023) * Ghostbuster script * Generate ghost list from API * Write scheduled workflow * Update logging * Support opening a PR * Move git config to where it’s needed * Update schedule to daily Co-authored-by: Ryan Cavanaugh <ryanca@microsoft.com>
burhon97 · Jan 11, 2022 · 2aa03ae · 2aa03ae
1 parent 0918ea4
commit 2aa03ae
Show file tree

Hide file tree

Showing 4 changed files with 207 additions and 0 deletions.
diff --git a/.github/workflows/ghostbuster.yml b/.github/workflows/ghostbuster.yml
@@ -0,0 +1,55 @@
+name: Remove contributors with deleted accounts
+
+on:
+  schedule:
+    # https://crontab.guru/#0_0_*_*_*
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+    inputs:
+      skipPR:
+        description: Push results straight to master instead of opening a PR
+        required: false
+        default: "false"
+
+jobs:
+  ghostbust:
+    runs-on: ubuntu-latest
+    if: github.repository == 'DefinitelyTyped/DefinitelyTyped'
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-node@v2
+
+      - run: npm ci
+      - run: node ./scripts/ghostbuster.cjs
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+
+      - if: ${{ inputs.skipPR == "true" }}
+        run: |
+          if [ -n "`git status -s`" ]; then
+            git config --global user.email "typescriptbot@microsoft.com"
+            git config --global user.name "TypeScript Bot"
+            git commit -am "Remove contributors with deleted accounts #no-publishing-comment"
+            # Script can take a bit to run; with such an active repo there's a good chance
+            # someone has merged a PR in that time.
+            git pull --rebase
+            git push
+          fi
+
+      - if: ${{ inputs.skipPR != "true" }}
+        uses: peter-evans/create-pull-request@v3.12.0
+        with:
+          token: ${{ secrets.GH_TOKEN }}
+          commit-message: "Remove contributors with deleted accounts #no-publishing-comment"
+          committer: "TypeScript Bot <typescriptbot@microsoft.com>"
+          author: "TypeScript Bot <typescriptbot@microsoft.com>"
+          branch: "bust-ghosts"
+          branch-suffix: short-commit-hash
+          delete-branch: true
+          title: Remove contributors with deleted accounts
+          body: "Generated from [.github/workflows/ghostbuster.yml](https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/.github/workflows/ghostbuster.yml)"
+
diff --git a/.gitignore b/.gitignore
@@ -35,6 +35,7 @@ _infrastructure/tests/build
 !*.js/
 !scripts/not-needed.cjs
 !scripts/close-old-issues.cjs
+!scripts/ghostbuster.cjs
 !scripts/remove-empty.cjs
 !scripts/update-codeowners.cjs
 

diff --git a/package.json b/package.json
@@ -28,7 +28,9 @@
         "@definitelytyped/definitions-parser": "latest",
         "@definitelytyped/dtslint": "latest",
         "@definitelytyped/dtslint-runner": "latest",
+        "@definitelytyped/header-parser": "^0.0.100",
         "@definitelytyped/utils": "latest",
+        "@octokit/core": "^3.5.1",
         "@octokit/rest": "^16.0.0",
         "d3-array": "^3.0.2",
         "d3-axis": "^3.0.0",

diff --git a/scripts/ghostbuster.cjs b/scripts/ghostbuster.cjs
@@ -0,0 +1,149 @@
+// @ts-check
+const { flatMap, mapDefined } = require('@definitelytyped/utils');
+const os = require('node:os');
+const path = require("path");
+const { writeFileSync, readFileSync, readdirSync, existsSync } = require('fs-extra');
+const hp = require("@definitelytyped/header-parser");
+const { Octokit } = require('@octokit/core');
+
+/**
+ * @param {string} indexPath
+ * @param {hp.Header & { raw: string }} header
+ * @param {Set<string>} ghosts
+ */
+function bust(indexPath, header, ghosts) {
+    /** @param {hp.Author} c */
+    const isGhost = c => c.githubUsername && ghosts.has(c.githubUsername.toLowerCase());
+    if (header.contributors.some(isGhost)) {
+        console.log(`Found one or more deleted accounts in ${indexPath}. Patching...`);
+        const indexContent = header.raw;
+        let newContent = indexContent;
+        if (header.contributors.length === 1) {
+            const prevContent = newContent;
+            newContent = newContent.replace(/^\/\/ Definitions by:.*$/mi, "// Definitions by: DefinitelyTyped <https://github.com/DefinitelyTyped>");
+            if (prevContent === newContent) throw new Error("Patch failed.");
+        } else {
+            const newOwnerList = header.contributors.filter(c => !isGhost(c));
+            if (newOwnerList.length === header.contributors.length) throw new Error("Didn't remove anyone??");
+            let newDefinitionsBy = `// Definitions by: ${newOwnerList[0].name} <https://github.com/${newOwnerList[0].githubUsername}>\n`;
+            for (let i = 1; i < newOwnerList.length; i++) {
+                newDefinitionsBy = newDefinitionsBy + `//                 ${newOwnerList[i].name} <https://github.com/${newOwnerList[i].githubUsername}>\n`;
+            }
+            const patchStart = newContent.indexOf("// Definitions by:");
+            const patchEnd = newContent.indexOf("// Definitions:");
+            if (patchStart === -1) throw new Error("No Definitions by:");
+            if (patchEnd === -1) throw new Error("No Definitions:");
+            if (patchEnd < patchStart) throw new Error("Definition header not in expected order");
+            newContent = newContent.substring(0, patchStart) + newDefinitionsBy + newContent.substring(patchEnd);
+        }
+
+        if (newContent !== indexContent) {
+            writeFileSync(indexPath, newContent, "utf-8");
+        }
+    }
+}
+
+/**
+ * @param {string} dir
+ * @param {(subpath: string) => void} fn
+ */
+function recurse(dir, fn) {
+    const entryPoints = readdirSync(dir, { withFileTypes: true })
+    for (const subdir of entryPoints) {
+        if (subdir.isDirectory() && subdir.name !== "node_modules") {
+            const subpath = path.join(dir, subdir.name);
+            fn(subpath);
+            recurse(subpath, fn);
+        }
+    }
+}
+
+function getAllHeaders() {
+    /** @type {Record<string, hp.Header & { raw: string }>} */
+    const headers = {};
+    console.log("Reading headers...");
+    recurse(path.join(__dirname, "../types"), subpath => {
+        const index = path.join(subpath, "index.d.ts");
+        if (existsSync(index)) {
+            const indexContent = readFileSync(index, "utf-8");
+            let parsed;
+            try {
+                parsed = hp.parseHeaderOrFail(indexContent);
+            } catch (e) {}
+            if (parsed) {
+                headers[index] = { ...parsed, raw: indexContent };
+            }
+        }
+    });
+    return headers;
+}
+
+/**
+ * @param {Set<string>} users
+ */
+async function fetchGhosts(users) {
+    console.log("Checking for deleted accounts...");
+    const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
+    const maxPageSize = 2000;
+    const pages = Math.ceil(users.size / maxPageSize);
+    const userArray = Array.from(users);
+    const ghosts = [];
+    for (let page = 0; page < pages; page++) {
+        const startIndex = page * maxPageSize;
+        const endIndex = Math.min(startIndex + maxPageSize, userArray.length);
+        const query = `query {
+            ${userArray.slice(startIndex, endIndex).map((user, i) => `u${i}: user(login: "${user}") { id }`).join("\n")}
+        }`;
+        const result = await tryGQL(() => octokit.graphql(query));
+        for (const k in result.data) {
+            if (result.data[k] === null) {
+                ghosts.push(userArray[startIndex + parseInt(k.substring(1), 10)]);
+            }
+        }
+    }
+
+    // Filter out organizations
+    const query = `query {
+        ${ghosts.map((user, i) => `o${i}: organization(login: "${user}") { id }`).join("\n")}
+    }`;
+    const result = await tryGQL(() => octokit.graphql(query));
+    return new Set(ghosts.filter(g => result.data[`o${ghosts.indexOf(g)}`] === null));
+}
+
+/**
+ * @param {() => Promise<any>} fn
+ */
+async function tryGQL(fn) {
+    try {
+        const result = await fn();
+        return result;
+    } catch (resultWithErrors) {
+        if (resultWithErrors.data) {
+            return resultWithErrors;
+        }
+        throw resultWithErrors;
+    }
+}
+
+process.on("unhandledRejection", err => {
+    console.error(err);
+    process.exit(1);
+});
+
+(async () => {
+    if (!process.env.GITHUB_TOKEN) {
+        throw new Error("GITHUB_TOKEN environment variable is not set");
+    }
+
+    const headers = getAllHeaders();
+    const users = new Set(flatMap(Object.values(headers), h => mapDefined(h.contributors, c => c.githubUsername?.toLowerCase())));
+    const ghosts = await fetchGhosts(users);
+    if (!ghosts.size) {
+        console.log("No ghosts found");
+        return;
+    }
+
+    for (const indexPath in headers) {
+        bust(indexPath, headers[indexPath], ghosts);
+    }
+})();