diff --git a/packages/scope-manager/src/referencer/Referencer.ts b/packages/scope-manager/src/referencer/Referencer.ts
index 06c19b67e455..4d582baf6d9e 100644
--- a/packages/scope-manager/src/referencer/Referencer.ts
+++ b/packages/scope-manager/src/referencer/Referencer.ts
@@ -4,6 +4,7 @@ import { AST_NODE_TYPES } from '@typescript-eslint/types';
 
 import type { GlobalScope, Scope } from '../scope';
 import type { ScopeManager } from '../ScopeManager';
+import type { ImplicitLibVariableOptions, Variable } from '../variable';
 import type { ReferenceImplicitGlobal } from './Reference';
 import type { VisitorOptions } from './Visitor';
 
@@ -19,6 +20,8 @@ import {
   VariableDefinition,
 } from '../definition';
 import { lib as TSLibraries } from '../lib';
+import { TYPE, VALUE, TYPE_VALUE } from '../lib/base-config';
+import { ImplicitLibVariable } from '../variable';
 import { ClassVisitor } from './ClassVisitor';
 import { ExportVisitor } from './ExportVisitor';
 import { ImportVisitor } from './ImportVisitor';
@@ -33,6 +36,15 @@ interface ReferencerOptions extends VisitorOptions {
   lib: Lib[];
 }
 
+type ImplicitVariableMap = Map<string, ImplicitLibVariableOptions>;
+
+// The `TSLibraries` object should have this type instead, but that's outside the scope of this prototype
+const entriesByLib = new Map<Lib, [string, ImplicitLibVariableOptions][]>();
+// This object caches by the serialized JSON representation of the file's libs array
+const variablesFromSerializedLibs = new Map<string, ImplicitVariableMap>();
+// This object caches by reference from the file's libs array
+const variablesFromRawLibs = new WeakMap<Lib[], ImplicitVariableMap>();
+
 // Referencing variables and creating bindings.
 class Referencer extends Visitor {
   #hasReferencedJsxFactory = false;
@@ -51,23 +63,130 @@ class Referencer extends Visitor {
   }
 
   private populateGlobalsFromLib(globalScope: GlobalScope): void {
-    for (const lib of this.#lib) {
-      const variables = TSLibraries[lib];
-      // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-      /* istanbul ignore if */ if (!variables) {
-        throw new Error(`Invalid value for lib provided: ${lib}`);
+    let implicitVariablesFromLibs: ImplicitVariableMap | undefined =
+      variablesFromRawLibs.get(this.#lib);
+    if (!implicitVariablesFromLibs) {
+      // Try to find out if this is a different object but same content we've seen before.
+      const libs: Lib[] = [...this.#lib].sort();
+      const serialized: string = JSON.stringify(libs);
+      implicitVariablesFromLibs = variablesFromSerializedLibs.get(serialized);
+      if (!implicitVariablesFromLibs) {
+        // No match, need to create it
+        implicitVariablesFromLibs = new Map();
+        for (const lib of libs) {
+          let entriesForLib:
+            | [string, ImplicitLibVariableOptions][]
+            | undefined = entriesByLib.get(lib);
+          if (!entriesForLib) {
+            const variables = TSLibraries[lib];
+            // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+            /* istanbul ignore if */ if (!variables) {
+              throw new Error(`Invalid value for lib provided: ${lib}`);
+            }
+            entriesForLib = Object.entries(variables);
+            entriesByLib.set(lib, entriesForLib);
+          }
+
+          for (const [name, variable] of entriesForLib) {
+            const existing = implicitVariablesFromLibs.get(name);
+            // Since a variable can be a type, a value, or both, and it isn't
+            // guaranteed to be the same between libs, merges
+            if (existing) {
+              if (existing === TYPE_VALUE || existing === variable) {
+                continue;
+              } else if (
+                (existing === VALUE && variable === TYPE) ||
+                (existing === TYPE && variable === VALUE)
+              ) {
+                implicitVariablesFromLibs.set(name, TYPE_VALUE);
+                continue;
+              }
+            }
+            // variable is either net-new, or is upgrading to TYPE_VALUE
+            implicitVariablesFromLibs.set(name, variable);
+          }
+        }
+        variablesFromSerializedLibs.set(serialized, implicitVariablesFromLibs);
+      }
+      variablesFromRawLibs.set(this.#lib, implicitVariablesFromLibs);
+    }
+
+    // Collect all names that are defined or referenced.
+    const allScopes = new Set<Scope>([globalScope]);
+    const allNames = new Set<string>();
+
+    for (const scope of allScopes) {
+      for (const childScope of scope.childScopes) {
+        allScopes.add(childScope);
       }
-      for (const [name, variable] of Object.entries(variables)) {
-        globalScope.defineImplicitVariable(name, variable);
+
+      for (const reference of scope.references) {
+        allNames.add(reference.identifier.name);
+      }
+
+      for (const variable of scope.variables) {
+        allNames.add(variable.name);
       }
     }
 
-    // for const assertions (`{} as const` / `<const>{}`)
-    globalScope.defineImplicitVariable('const', {
-      eslintImplicitGlobalSetting: 'readonly',
-      isTypeVariable: true,
-      isValueVariable: false,
-    });
+    // Rules only care about implicit globals if they are referenced or overlap with a local declaration,
+    // so add only those implicit globals.
+    const replacements = new Map<Variable, Variable>();
+    const arraysToProcess = new Set<Variable[]>([globalScope.variables]);
+    const { declaredVariables } = this.scopeManager;
+    for (const name of allNames) {
+      const libVariable: ImplicitLibVariableOptions | undefined =
+        implicitVariablesFromLibs.get(name);
+      if (libVariable) {
+        const existingVariable = globalScope.set.get(name);
+        if (existingVariable) {
+          // This should be cleaned up
+          const newVariable = new ImplicitLibVariable(
+            globalScope,
+            name,
+            libVariable,
+          );
+          replacements.set(existingVariable, newVariable);
+          globalScope.set.set(name, newVariable);
+
+          // Copy over information about the conflicting declaration
+          for (const def of existingVariable.defs) {
+            newVariable.defs.push(def);
+            // These arrays were created during defineVariable invocation
+            const parentDeclarations =
+              def.parent && declaredVariables.get(def.parent);
+            if (parentDeclarations) {
+              arraysToProcess.add(parentDeclarations);
+            }
+
+            const nodeDeclarations = declaredVariables.get(def.node);
+            if (nodeDeclarations) {
+              arraysToProcess.add(nodeDeclarations);
+            }
+          }
+
+          // This mapping is unidirectional, so just copy
+          for (const identifier of existingVariable.identifiers) {
+            newVariable.identifiers.push(identifier);
+          }
+
+          // References should not have been bound yet, so no action needed
+        } else {
+          globalScope.defineImplicitVariable(name, libVariable);
+        }
+      }
+    }
+
+    if (replacements.size > 0) {
+      for (const array of arraysToProcess) {
+        for (let i = array.length - 1; i >= 0; i--) {
+          const replacement = replacements.get(array[i]);
+          if (replacement) {
+            array[i] = replacement;
+          }
+        }
+      }
+    }
   }
   public close(node: TSESTree.Node): void {
     while (this.currentScope(true) && node === this.currentScope().block) {
@@ -569,7 +688,14 @@ class Referencer extends Visitor {
 
   protected Program(node: TSESTree.Program): void {
     const globalScope = this.scopeManager.nestGlobalScope(node);
-    this.populateGlobalsFromLib(globalScope);
+
+    // Keep this first since it reduces the amount of snapshot changes significantly
+    // for const assertions (`{} as const` / `<const>{}`)
+    globalScope.defineImplicitVariable('const', {
+      eslintImplicitGlobalSetting: 'readonly',
+      isTypeVariable: true,
+      isValueVariable: false,
+    });
 
     if (this.scopeManager.isGlobalReturn()) {
       // Force strictness of GlobalScope to false when using node.js scope.
@@ -586,6 +712,9 @@ class Referencer extends Visitor {
     }
 
     this.visitChildren(node);
+    // Need to call this after all references and variables are defined,
+    // but before we bind the global scope's references.
+    this.populateGlobalsFromLib(globalScope);
     this.close(node);
   }
 
diff --git a/packages/scope-manager/tests/lib.test.ts b/packages/scope-manager/tests/lib.test.ts
index eb558f3eb6e9..1774374094cf 100644
--- a/packages/scope-manager/tests/lib.test.ts
+++ b/packages/scope-manager/tests/lib.test.ts
@@ -11,15 +11,110 @@ describe('implicit lib definitions', () => {
     expect(variables).toHaveLength(1);
   });
 
-  it('should define implicit variables', () => {
-    const { scopeManager } = parseAndAnalyze('', {
+  it('should define an implicit variable if there is a value reference', () => {
+    const { scopeManager } = parseAndAnalyze('new ArrayBuffer();', {
+      lib: ['es2015'],
+    });
+
+    const variables = scopeManager.variables;
+    const arrayBufferVariables = variables.filter(
+      v => v.name === 'ArrayBuffer',
+    );
+    expect(arrayBufferVariables).toHaveLength(1);
+    expect(arrayBufferVariables[0]).toBeInstanceOf(ImplicitLibVariable);
+  });
+
+  it('should define an implicit variable if there is a type reference', () => {
+    const { scopeManager } = parseAndAnalyze('type T = ArrayBuffer;', {
+      lib: ['es2015'],
+    });
+
+    const variables = scopeManager.variables;
+    const arrayBufferVariables = variables.filter(
+      v => v.name === 'ArrayBuffer',
+    );
+    expect(arrayBufferVariables).toHaveLength(1);
+    expect(arrayBufferVariables[0]).toBeInstanceOf(ImplicitLibVariable);
+  });
+
+  it('should define an implicit variable if there is a nested value reference', () => {
+    const { scopeManager } = parseAndAnalyze(
+      'var f = () => new ArrayBuffer();',
+      {
+        lib: ['es2015'],
+      },
+    );
+
+    const variables = scopeManager.variables;
+    const arrayBufferVariables = variables.filter(
+      v => v.name === 'ArrayBuffer',
+    );
+    expect(arrayBufferVariables).toHaveLength(1);
+    expect(arrayBufferVariables[0]).toBeInstanceOf(ImplicitLibVariable);
+  });
+
+  it('should define an implicit variable if there is a nested type reference', () => {
+    const { scopeManager } = parseAndAnalyze(
+      'var f = <T extends ArrayBuffer>(): T => undefined as T;',
+      {
+        lib: ['es2015'],
+      },
+    );
+
+    const variables = scopeManager.variables;
+    const arrayBufferVariables = variables.filter(
+      v => v.name === 'ArrayBuffer',
+    );
+    expect(arrayBufferVariables).toHaveLength(1);
+    expect(arrayBufferVariables[0]).toBeInstanceOf(ImplicitLibVariable);
+  });
+
+  it('should define an implicit variable if there is a value collision', () => {
+    const { scopeManager } = parseAndAnalyze('var Symbol = 1;', {
+      lib: ['es2015'],
+    });
+
+    const variables = scopeManager.variables;
+    const symbolVariables = variables.filter(v => v.name === 'Symbol');
+    expect(symbolVariables).toHaveLength(1);
+    expect(symbolVariables[0]).toBeInstanceOf(ImplicitLibVariable);
+  });
+
+  it('should define an implicit variable if there is a type collision', () => {
+    const { scopeManager } = parseAndAnalyze('type Symbol = 1;', {
       lib: ['es2015'],
     });
 
     const variables = scopeManager.variables;
-    expect(variables.length).toBeGreaterThan(1);
+    const symbolVariables = variables.filter(v => v.name === 'Symbol');
+    expect(symbolVariables).toHaveLength(1);
+    expect(symbolVariables[0]).toBeInstanceOf(ImplicitLibVariable);
+  });
 
-    const variable = variables[0];
-    expect(variable).toBeInstanceOf(ImplicitLibVariable);
+  it('should define an implicit variable if there is a nested value collision', () => {
+    const { scopeManager } = parseAndAnalyze('var f = (Symbol) => Symbol;', {
+      lib: ['es2015'],
+    });
+
+    const variables = scopeManager.variables;
+    const symbolVariables = variables.filter(v => v.name === 'Symbol');
+    expect(symbolVariables).toHaveLength(2);
+    expect(symbolVariables.some(v => v instanceof ImplicitLibVariable)).toBe(
+      true,
+    );
+    expect(symbolVariables.some(v => !(v instanceof ImplicitLibVariable))).toBe(
+      true,
+    );
+  });
+
+  it('should define an implicit variable if there is a nested type collision', () => {
+    const { scopeManager } = parseAndAnalyze('var f = (a: Symbol) => a;', {
+      lib: ['es2015'],
+    });
+
+    const variables = scopeManager.variables;
+    const symbolVariables = variables.filter(v => v.name === 'Symbol');
+    expect(symbolVariables).toHaveLength(1);
+    expect(symbolVariables[0]).toBeInstanceOf(ImplicitLibVariable);
   });
 });