Async Map with Highland.js

Question

I have a Highland stream that is periodically getting data from a server. I need to do a database lookup inside of a map. I can't find any mention of doing anything async in any of Highland's transformers.

Answer 1

You can use consume to process a stream in an async manner.

_([1, 2, 3, 4]).consume(function(err, item, push, next) {
  // Do fancy async thing
  setImmediate(function() {
    // Push the number onto the new stream
    push(null, item);

    // Consume the next item
    next();
  });
})).toArray(function(items) {
  console.log(items); // [1, 2, 3, 4]
});

Answer 2

After using a .map you can use a .sequence as:

var delay = _.wrapCallback(function delay(num, cb){
    setTimeout(function(){ cb(null, num+1); }, 1000);
});

_([1,2,3,4,5]).map(function(num){
    return delay(num);
}).sequence().toArray(function(arr){ // runs one by one here
    console.log("Got xs!", arr);
});

Fiddle here.

Or with .parallel in parallel:

var delay = _.wrapCallback(function delay(num, cb){
    setTimeout(function(){ cb(null, num+1); }, 1000);
});

_([1,2,3,4,5]).map(function(num){
    console.log("got here", num);
    return delay(num);
}).parallel(10).toArray(function(arr){ // 10 at a time
    console.log("Got xs!", arr);
});

Fiddle here