I can't complete any http request with GOT nor Node's http module on certain website and the server is responding with status code 403.
However, while using the puppeteer it completes the request and responds with status 200.
I set all cookies and headers the same in both methods, any clues as to what's going on?
https://codepen.io/kuba-maliszewski/pen/oNYvLJG
const puppeteer = require('puppeteer-extra')
const { useProxy } = require('puppeteer-page-proxy')
const got = require('got')
const { CookieJar } = require('tough-cookie')
const cookieJar = new CookieJar()
const setCookie = promisify(cookieJar.setCookie.bind(cookieJar))
;(async () => {
const browser = await puppeteer.launch({
headless: false,
args: ['--disable-web-security'],
})
const page = await browser.newPage()
await page.goto('https://example.com', { waitUntil: 'networkidle0' })
const cookies = await page.cookies()
cookies.forEach(async (cookie) => {
await setCookie(`${cookie.name}=${cookie.value}`, page.url(), [{ secure: true }])
})
const options = {
cookieJar: cookieJar,
method: 'POST',
body: `{"simpleSku":"${bodyHTML.model.groups[0].articles[0].simpleSku}", "ids":[]}`, //bodyHTML is defined somewhere in code
headers: Object.assign({}, headers, { //headers are the same as in working request - ALREADY CHECKED
accept: 'application/json',
'accept-encoding': 'gzip, deflate, br',
host: 'www.example.com',
'content-type': 'application/json',
dpr: '1',
origin: 'https://example.com',
'accept-encoding': 'gzip, deflate, br',
'upgrade-insecure-requests': '1',
referer: 'https://example.com',
}),
responseType: 'buffer',
maxRedirects: 15,
throwHttpErrors: false,
ignoreInvalidCookies: true,
followRedirect: false,
}
const response = await got('https://example.com', options) //THIS DOESN'T WORK, resulting in status code 403
page.on('request', async (request) => {
const options = {
method: 'POST',
postData: `{"simpleSku":"${bodyHTML.model.groups[0].articles[0].simpleSku}", "ids":[]}`, //bodyHTML is defined somewhere in code
headers: Object.assign({}, request.headers(), headers), //headers are the same as in working request - ALREADY CHECKED
}
request.continue(options)
})
let res1 = await page.goto('https://example.com') //THIS WORKS, resulting in status code 200
})()
question from:
https://stackoverflow.com/questions/65926779/puppeteer-request-vs-got-request 与恶龙缠斗过久,自身亦成为恶龙;凝视深渊过久,深渊将回以凝视…