I'm starting a new Django project, I'm trying to capture the network traffic with Selenium.
I have already achieved this objective with Selenium-wire (MITM Proxy), but Django doesn't like to work with selenium-wire ( must start the server with "--nothreading --noreload", connection bug... ).
I'm looking for achieve this with modern solutions, like parsing the network devtools of firefox directly or with a firefox addons.
I'm using Firefox Geckodriver for my test.
for x in range(0, 10):
profile = profile_firefox()
options = options_firefox()
driver = webdriver.Firefox(firefox_profile=profile, options=options, executable_path='/Users/*****/Desktop/selenium-master/headless_browser/geckodriver')
try:
driver.set_window_position(0, 0)
driver.set_window_size(randint(1024, 2060), randint(1024, 4100))
time.sleep(randint(3,10))
driver.get(url)
wait = ui.WebDriverWait(driver, 10)
time.sleep(randint(8,10))
if driver.find_element_by_xpath("//*[@id="container"]/main/div/div/div[1]/div[2]/form/div/div[2]/div[1]/button"):
driver.find_element_by_xpath("//*[@id="container"]/main/div/div/div[1]/div[2]/form/div/div[2]/div[1]/button").click()
del driver.requests
time.sleep(randint(8,10))
driver.find_element_by_xpath("//*[@id="container"]/main/div/div/div[1]/div[2]/form/div/div[2]/div[1]/button").click()
time.sleep(randint(10,20))
for request in driver.requests:
if request.path == "https://api.*********.**/*******/*******":
request_api = request
raw = str(request_api.body)
request_api = raw.split(('b''))
payload_raw = request_api[1]
payload = payload_raw[:-1]
if payload:
header = request.headers
time.sleep(8)
break
except:
print("Houston on a eu un probleme")
firefox_closing(driver)
Edit :
def profile_firefox():
profile = FirefoxProfile()
profile.set_preference('permissions.default.image', 2)
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
profile.set_preference("general.useragent.override", firefox_init())
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.socks', 'localhost')
profile.set_preference('network.proxy.socks_port', 9050)
profile.set_preference("network.proxy.socks_remote_dns", False)
profile.set_preference("driver.privatebrowsing.autostart", True)
profile.update_preferences()
return profile
Test 2 with Socks,HTTP,SSL configuration :
server = Server('/Users/*****/Desktop/selenium-master/browsermob-proxy-2.1.4/bin/browsermob-proxy')
server.start()
proxy = server.create_proxy()
proxy.selenium_proxy()#Dont understand what it does ???
port = int(proxy.port)
profile = FirefoxProfile()
profile.set_preference('permissions.default.image', 2)
profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
profile.set_preference('general.useragent.override', firefox_init())
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.socks', 'localhost')
profile.set_preference('network.proxy.socks_port', 9050)
profile.set_preference('network.proxy.ssl', 'localhost')
profile.set_preference('network.proxy.ssl_port', port)
profile.set_preference('network.proxy.http', 'localhost')
profile.set_preference('network.proxy.http_port', port)
profile.set_preference('network.proxy.socks_remote_dns', False)
profile.set_preference('driver.privatebrowsing.autostart', True)
profile.update_preferences()
It seems Http proxy override the socks configuration...
Thanks a lot if you have any clue or advice about my code or solutions.
See Question&Answers more detail:
os