diff --git a/packages/type-utils/src/typeOrValueSpecifiers/typeDeclaredInPackageDeclarationFile.ts b/packages/type-utils/src/typeOrValueSpecifiers/typeDeclaredInPackageDeclarationFile.ts
index 980128f9be81..35090c59ee7e 100644
--- a/packages/type-utils/src/typeOrValueSpecifiers/typeDeclaredInPackageDeclarationFile.ts
+++ b/packages/type-utils/src/typeOrValueSpecifiers/typeDeclaredInPackageDeclarationFile.ts
@@ -1,3 +1,5 @@
+import { getCanonicalFileName } from '@typescript-eslint/typescript-estree';
+import path from 'node:path';
 import * as ts from 'typescript';
 
 function findParentModuleDeclaration(
@@ -30,15 +32,36 @@ function typeDeclaredInDeclarationFile(
   declarationFiles: ts.SourceFile[],
   program: ts.Program,
 ): boolean {
-  // Handle scoped packages: if the name starts with @, remove it and replace / with __
-  const typesPackageName = packageName.replace(/^@([^/]+)\//, '$1__');
+  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_expressions#escaping
+  const escapeAlternates = (alternates: string[]): string =>
+    alternates
+      .map(alternate => alternate.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&'))
+      .join('|');
 
-  const matcher = new RegExp(`${packageName}|${typesPackageName}`);
+  const packageNameRegExp = escapeAlternates([
+    packageName,
+    // Handle scoped packages: if the name starts with @, remove it and replace / with __
+    `@types/${packageName.replace(/^@([^/]+)\//, '$1__')}`,
+  ]);
+  const packageNameMatcher = new RegExp(packageNameRegExp);
+  const { typeRoots } = program.getCompilerOptions();
+  const fileNameMatcher = new RegExp(
+    `(?:${escapeAlternates(
+      typeRoots
+        ? typeRoots.map(typeRoot =>
+            getCanonicalFileName(
+              path.join(program.getCurrentDirectory(), typeRoot),
+            ),
+          )
+        : ['node_modules'],
+    )})/(?:${packageNameRegExp})/`,
+  );
   return declarationFiles.some(declaration => {
     const packageIdName = program.sourceFileToPackageName.get(declaration.path);
     return (
-      packageIdName !== undefined &&
-      matcher.test(packageIdName) &&
+      (packageIdName == null
+        ? fileNameMatcher.test(getCanonicalFileName(declaration.fileName))
+        : packageNameMatcher.test(packageIdName)) &&
       program.isSourceFileFromExternalLibrary(declaration)
     );
   });
diff --git a/packages/type-utils/tests/TypeOrValueSpecifier.test.ts b/packages/type-utils/tests/TypeOrValueSpecifier.test.ts
index 7af6d7dac2b0..5743bc05e4b3 100644
--- a/packages/type-utils/tests/TypeOrValueSpecifier.test.ts
+++ b/packages/type-utils/tests/TypeOrValueSpecifier.test.ts
@@ -4,7 +4,7 @@ import { parseForESLint } from '@typescript-eslint/parser';
 import Ajv from 'ajv';
 import path from 'node:path';
 
-import type { TypeOrValueSpecifier } from '../src/TypeOrValueSpecifier';
+import type { TypeOrValueSpecifier } from '../src';
 
 import { typeMatchesSpecifier, typeOrValueSpecifiersSchema } from '../src';
 
@@ -366,6 +366,17 @@ describe('TypeOrValueSpecifier', () => {
           package: 'node:test',
         },
       ],
+      [
+        `
+          import { Buffer } from 'node:buffer';
+          type Test = Buffer;
+        `,
+        { from: 'package', name: 'Buffer', package: 'node' },
+      ],
+      [
+        'type Test = Buffer;',
+        { from: 'package', name: 'Buffer', package: 'node' },
+      ],
     ])('matches a matching package specifier: %s', runTestPositive);
 
     it.each<[string, TypeOrValueSpecifier]>([