Node.js Proxy with custom Http(s) Agent and Connect.js Middleware

Question

I've put together a proxy server in Node that needs the ability to tunnel https requests over tls and that all works. Using the the following two packages this was extremely easy to setup: proxy, https-proxy-agent. My issue is that I'm trying to capture HAR files using connect as a middleware layer and I'm getting the following error:

_http_outgoing.js:357
throw new Error('Can\'t set headers after they are sent.');
^
Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (_http_outgoing.js:357:11)
at ServerResponse.writeHead (_http_server.js:180:21)
at ClientRequest.<anonymous> (/.../node_modules/proxy/proxy.js:233:11)
at emitOne (events.js:96:13)
at ClientRequest.emit (events.js:188:7)
at HTTPParser.parserOnIncomingClient [as onIncoming] (_http_client.js:473:21)
at HTTPParser.parserOnHeadersComplete (_http_common.js:99:23)
at Socket.socketOnData (_http_client.js:362:20)
at emitOne (events.js:96:13)
at Socket.emit (events.js:188:7)

The is as simple as the following and it only seems to happen when I'm using connect and proxying through my local browser(this proxy is actually being used with BrowserStackLocal). When I pass through the proxy from anything other than my local machines browser, it's like it doesn't even know the middleware exists.

So basically, I just need to get connect working in this scenario and I'm not sure if I need to pause something and resume, or what... any ideas would be greatly appreciated. The base code is below:

const path = require('path');
const http = require('http');
const proxy = require('proxy');
const Agent = require('https-proxy-agent');
const connect = require('connect');
const har = require('./har');

const middleware = connect();

middleware.use(har({
  harOutputDir: path.resolve(process.cwd(), 'har/')
}));

const server = proxy(http.createServer(middleware));
server.agent = new Agent('http://localhost:8081');

server.listen(8081)

Thanks!

EDIT: Just a note: the har middleware is not modifying headers at all.

Answer 1

proxy hasn't been maintained in a while. Builds are not passing, last commit don't pass tests. The source of the stack trace you've put up is coming from here in Line 233 - buggy library code.

Writing a similar proxy is trivial. Following code illustrates how to create one.

const http = require('http');
const urlP = require('url');

const proxiedServer = 'http://localhost:8888';

// Proxy server
http.createServer((req, res) => {
  console.log(`Proxy: Got ${req.url}`);

  const _r = http.request(
    Object.assign(
      {},
      urlP.parse(proxiedServer),
      {
        method: req.method,
        path: req.url
      }
    ),
    _res => {
      res.writeHead(_res.statusCode, _res.headers);
      _res.pipe(res);
    }
  );

  req.pipe(_r);

}).listen(3124, () => {
  console.log("Listening on 3124");
});


// Regular server. Could be Express
http.createServer((req, res) => {
  console.log('Proxied server: ', req.url);
  let b = '';
  req.on('data', c => {
    b += c;
  });
  req.on('end', () => {
    console.log('Proxied server: ', b);
  });
  res.writeHead(200);
  res.end('ok');
}).listen(8888, () => {
  console.log('proxied server listening on 8888');
});

Your code using your own custom proxy would look like the following:

const urlP = require('url');
const path = require('path');
const http = require('http');
const connect = require('connect');
const har = require('./har');

const proxiedServer = 'http://localhost:8888';

// Proxy server
http.createServer((req, res) => {
  console.log(`Proxy: Got ${req.url}`);

  const _r = http.request(
    Object.assign(
      {},
      urlP.parse(proxiedServer),
      {
        method: req.method,
        path: req.url
      }
    ),
    _res => {
      res.writeHead(_res.statusCode, _res.headers);
      _res.pipe(res);
    }
  );

  req.pipe(_r);

}).listen(3124, () => {
  console.log("Listening on 3124");
});

const middleware = connect();
middleware.use(har({
  harOutputDir: path.resolve(process.cwd(), 'har/')
}));

middleware.use((req, res) => {
  console.log('Proxied server: ', req.url);
  let b = '';
  req.on('data', c => {
    b += c;
  });
  req.on('end', () => {
    console.log('Proxied server: ', b);
  });
  res.writeHead(200);
  res.end('ok');
});


http.createServer(middleware).listen(8888, () => {
  console.log('proxied server listening on 8888');
});