Capture AJAX response with selenium and python

Question

I click on a link in Firefox, the webpage sends a request using javascript, then the server sends some sort of response which includes a website address. So this new website then opens in a new Window. The html code behind the link is (I've omitted initial and final <span> tag):

> class="taLnk hvrIE6"
> onclick="ta.trackEventOnPage('AttractionContactInfo', 'Website',
> 2316062, 1); ta.util.cookie.setPIDCookie(15190);
> ta.call('ta.util.link.targetBlank', event, this,
> {'aHref':'LqMWJQiMnYQQoqnQQxGEcQQoqnQQWJQzZYUWJQpEcYGII26XombQQoqnQQQQoqnqgoqnQQQQoqnQQQQoqnQQQQoqnqgoqnQQQQoqnQQuuuQQoqnQQQQoqnxioqnQQQQoqnQQJMsVCIpEVMSsVEtHJcSQQoqnQQQQoqnxioqnQQQQoqnQQniaQQoqnQQQQoqnqgoqnQQQQoqnQQWJQzhYmkXHJUokUHnmKTnJXB',
> 'isAsdf':true})">Website

I want to capture the server response and extract the 'new website' using Python and Selenium. I've been using BeautifulSoup for scraping and am pretty new to Selenium.

So far, I am able to find this element and click on it using selenium, which opens the 'new website' in a new window. I don't know how to capture the response from server.

Answer 1

I once intercepted some ajax calls injecting javascript to the page using selenium. The bad side of the history is that selenium could sometimes be, let's say "fragile". So for no reason I got selenium exceptions while doing this injection.

Anyway, my idea was intercept the XHR calls, and set its response to a new dom element created by me that I could manipulate from selenium. In the condition for the interception you can even use the url that made the request in order to just intercept the one that you actually want (self._url)

btw, I got the idea from intercept all ajax calls?

Maybe this helps.

browser.execute_script("""
(function(XHR) {
  "use strict";

  var element = document.createElement('div');
  element.id = "interceptedResponse";
  element.appendChild(document.createTextNode(""));
  document.body.appendChild(element);

  var open = XHR.prototype.open;
  var send = XHR.prototype.send;

  XHR.prototype.open = function(method, url, async, user, pass) {
    this._url = url; // want to track the url requested
    open.call(this, method, url, async, user, pass);
  };

  XHR.prototype.send = function(data) {
    var self = this;
    var oldOnReadyStateChange;
    var url = this._url;

    function onReadyStateChange() {
      if(self.status === 200 && self.readyState == 4 /* complete */) {
        document.getElementById("interceptedResponse").innerHTML +=
          '{"data":' + self.responseText + '}*****';
      }
      if(oldOnReadyStateChange) {
        oldOnReadyStateChange();
      }
    }

    if(this.addEventListener) {
      this.addEventListener("readystatechange", onReadyStateChange,
        false);
    } else {
      oldOnReadyStateChange = this.onreadystatechange;
      this.onreadystatechange = onReadyStateChange;
    }
    send.call(this, data);
  }
})(XMLHttpRequest);
""")