javascript - Method for streaming data from browser to server via HTTP

Question

Are there any XHR-like browser APIs available for streaming binary to a server over HTTP?

I want to make an HTTP PUT request and create data programmatically, over time. I don't want to create all this data at once, since there could be gigs of it sitting in memory. Some psueudo-code to illustrate what I'm getting at:

var dataGenerator = new DataGenerator(); // Generates 8KB UInt8Array every second
var streamToWriteTo;
http.put('/example', function (requestStream) {
  streamToWriteTo = requestStream;
});

dataGenerator.on('data', function (chunk) {
  if (!streamToWriteTo) {
    return;
  }
  streamToWriteTo.write(chunk);
});

I currently have a web socket solution in place instead, but would prefer regular HTTP for better interop with some existing server-side code.

EDIT: I can use bleeding edge browser APIs. I was looking at the Fetch API, as it supports ArrayBuffers, DataViews, Files, and such for request bodies. If I could somehow fake out one of these objects so that I could use the Fetch API with dynamic data, that would work for me. I tried creating a Proxy object to see if any methods were called that I could monkey patch. Unfortunately, it seems that the browser (at least in Chrome) is doing the reading in native code and not in JS land. But, please correct me if I'm wrong on that.

Answer 1

I don't know how to do this with pure HTML5 APIs, but one possible workaround is to use a Chrome App as a background service to provide additional features to a web page. If you're already willing to use development browsers and enable experimental features, this seems like just an incremental step further than that.

Chrome Apps can call the chrome.sockets.tcp API, on which you can implement any protocol you want, including HTTP and HTTPS. This would provide the flexibility to implement streaming.

A regular web page can exchange messages with an App using the chrome.runtime API, as long as the App declares this usage. This would allow your web page to make asynchronous calls to your App.

I wrote this simple App as a proof of concept:

manifest.json

{
  "manifest_version" : 2,

  "name" : "Streaming Upload Test",
  "version" : "0.1",

  "app": {
    "background": {
      "scripts": ["background.js"]
    }
  },

  "externally_connectable": {
    "matches": ["*://localhost/*"]
  },

  "sockets": {
    "tcp": {
      "connect": "*:*"
    }
  },

  "permissions": [
  ]
}

background.js

var mapSocketToPort = {};

chrome.sockets.tcp.onReceive.addListener(function(info) {
  var port = mapSocketToPort[info.socketId];
  port.postMessage(new TextDecoder('utf-8').decode(info.data));
});

chrome.sockets.tcp.onReceiveError.addListener(function(info) {
  chrome.sockets.tcp.close(info.socketId);
  var port = mapSocketToPort[info.socketId];
  port.postMessage();
  port.disconnect();
  delete mapSocketToPort[info.socketId];
});

// Promisify socket API for easier operation sequencing.
// TODO: Check for error and reject.
function socketCreate() {
  return new Promise(function(resolve, reject) {
    chrome.sockets.tcp.create({ persistent: true }, resolve);
  });
}

function socketConnect(s, host, port) {
  return new Promise(function(resolve, reject) {
    chrome.sockets.tcp.connect(s, host, port, resolve);
  });
}

function socketSend(s, data) {
  return new Promise(function(resolve, reject) {
    chrome.sockets.tcp.send(s, data, resolve);
  });
}

chrome.runtime.onConnectExternal.addListener(function(port) {
  port.onMessage.addListener(function(msg) {
    if (!port.state) {
      port.state = msg;

      port.chain = socketCreate().then(function(info) {
        port.socket = info.socketId;
        mapSocketToPort[port.socket] = port;
        return socketConnect(port.socket, 'httpbin.org', 80);
      }).then(function() {
        // TODO: Layer TLS if needed.
      }).then(function() {
        // TODO: Build headers from the request.
        // TODO: Use Transfer-Encoding: chunked.
        var headers =
            'PUT /put HTTP/1.0
' +
            'Host: httpbin.org
' +
            'Content-Length: 17
' +
            '
';
        return socketSend(port.socket, new TextEncoder('utf-8').encode(headers).buffer);
      });
    }
    else {
      if (msg) {
        port.chain = port.chain.then(function() {
          // TODO: Use chunked encoding.
          return socketSend(port.socket, new TextEncoder('utf-8').encode(msg).buffer);
        });
      }
    }
  });
});

This App does not have a user interface. It listens for connections and makes a hard-coded PUT request to http://httpbin.org/put (httpbin is a useful test site but note it does not support chunked encoding). The PUT data (currently hard-coded to exactly 17 octets) is streamed in from the client (using as few or as many messages as desired) and sent to the server. The response from the server is streamed back to the client.

This is just a proof of concept. A real App should probably:

• Connect to any host and port.
• Use Transfer-Encoding: chunked.
• Signal the end of streaming data.
• Handle socket errors.
• Support TLS (e.g. with Forge)

Here is a sample web page that performs a streaming upload (of 17 octets) using the App as a service (note that you will have to configure your own App id):

<pre id="result"></pre>
<script>
 var MY_CHROME_APP_ID = 'omlafihmmjpklmnlcfkghehxcomggohk';

 function streamingUpload(url, options) {
   // Open a connection to the Chrome App. The argument must be the 
   var port = chrome.runtime.connect(MY_CHROME_APP_ID);

   port.onMessage.addListener(function(msg) {
     if (msg)
       document.getElementById("result").textContent += msg;
     else
       port.disconnect();
   });

   // Send arguments (must be JSON-serializable).
   port.postMessage({
     url: url,
     options: options
   });

   // Return a function to call with body data.
   return function(data) {
     port.postMessage(data);
   };
 }

 // Start an upload.
 var f = streamingUpload('https://httpbin.org/put', { method: 'PUT' });

 // Stream data a character at a time.
 'how now brown cow'.split('').forEach(f);
</script>

When I load this web page in a Chrome browser with the App installed, httpbin returns:

HTTP/1.1 200 OK
Server: nginx
Date: Sun, 19 Jun 2016 16:54:23 GMT
Content-Type: application/json
Content-Length: 240
Connection: close
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true

{
  "args": {}, 
  "data": "how now brown cow", 
  "files": {}, 
  "form": {}, 
  "headers": {
    "Content-Length": "17", 
    "Host": "httpbin.org"
  }, 
  "json": null, 
  "origin": "[redacted]", 
  "url": "http://httpbin.org/put"
}