from bs4 import BeautifulSoup
import requests
import json
params = {
'id': '{"application":"WSJ","marketsDiaryType":"overview"}',
'type': 'mdc_marketsdiary'
}
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:73.0) Gecko/20100101 Firefox/73.0"
}
r = requests.get(
"https://www.wsj.com/market-data/stocks", params=params, headers=headers).json()
data = json.dumps(r, indent=4)
print(data)
Output:
{
"id": "{"application":"WSJ","marketsDiaryType":"overview"}",
"type": "mdc_marketsdiary",
"data": {
"instrumentSets": [
{
"headerFields": [
{
"value": "name",
"label": "Issues"
}
],
"instruments": [
{
"name": "Advancing",
"NASDAQ": "169",
"NYSE": "69"
},
{
"name": "Declining",
"NASDAQ": "3,190",
"NYSE": "2,973"
},
{
"name": "Unchanged",
"NASDAQ": "24",
"NYSE": "10"
},
{
"name": "Total",
"NASDAQ": "3,383",
"NYSE": "3,052"
}
]
},
{
"headerFields": [
{
"value": "name",
"label": "Issues At"
}
],
"instruments": [
{
"name": "New Highs",
"NASDAQ": "53",
"NYSE": "14"
},
{
"name": "New Lows",
"NASDAQ": "1,406",
"NYSE": "1,620"
}
]
},
{
"headerFields": [
{
"value": "name",
"label": "Share Volume"
}
],
"instruments": [
{
"name": "Total",
"NASDAQ": "4,454,691,895",
"NYSE": "7,790,947,818"
},
{
"name": "Advancing",
"NASDAQ": "506,192,012",
"NYSE": "219,412,232"
},
{
"name": "Declining",
"NASDAQ": "3,948,035,191",
"NYSE": "7,570,377,893"
},
{
"name": "Unchanged",
"NASDAQ": "464,692",
"NYSE": "1,157,693"
}
]
}
],
"timestamp": "4:00 PM EDT 3/09/20"
},
"hash": "{"id":"{"application":"WSJ","marketsDiaryType":"overview"}","type":"mdc_marketsdiary","data":{"instrumentSets":[{"headerFields":[{"value":"name","label":"Issues"}],"instruments":[{"name":"Advancing","NASDAQ":"169","NYSE":"69"},{"name":"Declining","NASDAQ":"3,190","NYSE":"2,973"},{"name":"Unchanged","NASDAQ":"24","NYSE":"10"},{"name":"Total","NASDAQ":"3,383","NYSE":"3,052"}]},{"headerFields":[{"value":"name","label":"Issues At"}],"instruments":[{"name":"New Highs","NASDAQ":"53","NYSE":"14"},{"name":"New Lows","NASDAQ":"1,406","NYSE":"1,620"}]},{"headerFields":[{"value":"name","label":"Share Volume"}],"instruments":[{"name":"Total","NASDAQ":"4,454,691,895","NYSE":"7,790,947,818"},{"name":"Advancing","NASDAQ":"506,192,012","NYSE":"219,412,232"},{"name":"Declining","NASDAQ":"3,948,035,191","NYSE":"7,570,377,893"},{"name":"Unchanged","NASDAQ":"464,692","NYSE":"1,157,693"}]}],"timestamp":"4:00 PM EDT 3/09/20"}}"
}
Note: You can access it as dict
print(r.keys())
.