add got-scraping options

JaneJeon · JaneJeon · commit 539363e9dace · 2021-12-24T20:12:03.000-05:00
diff --git a/index.js b/index.js
@@ -6,6 +6,7 @@ import dnsLookupGen from './utils/dns-lookup.js'
 import logger from './utils/logger.js'
 import CacheableLookup from 'cacheable-lookup'
 import { gotSsrf } from 'got-ssrf'
+import { PRESETS } from 'header-generator'
 
 const debug = logger('index.js')
 
@@ -22,7 +23,12 @@ export default (
       request: 14000 // global timeout
     },
     cache: new QuickLRU({ maxSize: 10000 }),
-    dnsCache: new CacheableLookup({ cache: new QuickLRU({ maxSize: 100000 }) })
+    dnsCache: new CacheableLookup({ cache: new QuickLRU({ maxSize: 100000 }) }),
+    context: {
+      insecureHTTPParser: false,
+      proxyUrl: process.env.HTTP_PROXY_URL,
+      headerGeneratorOptions: PRESETS.MODERN_WINDOWS_CHROME
+    }
   },
   timeoutMs = 15000,
   canonicizeMemOpts = { cache: new QuickLRU({ maxSize: 100000 }) },
diff --git a/utils/http-client.js b/utils/http-client.js
@@ -1,4 +1,4 @@
-import got from 'got'
+import { got } from 'got'
 import { gotScraping } from 'got-scraping'
 import { gotSsrf } from 'got-ssrf'
 import canonicizeHookGen from './canonicize.js'

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import got from 'got'`
	`1`	`+import { got } from 'got'`
`2`	`2`	`import { gotScraping } from 'got-scraping'`
`3`	`3`	`import { gotSsrf } from 'got-ssrf'`
`4`	`4`	`import canonicizeHookGen from './canonicize.js'`