Can't use promises & async wait in AWS Lambda node.js working with docClient & Firehose

Question

I am unable to understand how to reliably use async in node.js when working with firehose and dynamoDB functions. For example:

async function dynamoDBToCSV(record) {
    ... 
    try {
        const res = await firehose.putRecordBatch(params).promise();
        console.log('res = ', res);
    } catch (err) { 
        console.log('err = ', err);
    }
}

The code which calls the function:

let dataCount = 0;
let items;

do {
    let scanResults = [];
    items =  await docClient.scan(params).promise();
    items.Items.forEach(async(item) => {
        
        dataCount++;

        try {
            if (dataCount === 3) { // test on one packet. 
                await dynamoDBToCSV(item);
            }
            
        } catch(err) { 
            console.log('Failed at item number ' + dataCount);
            return err;
        }
    });
     
    params.ExclusiveStartKey  = items.LastEvaluatedKey;
// } while (typeof items.LastEvaluatedKey != "undefined");
} while (false); // test mode. Runs on one page.

console.log('Interventions in DB = ' + dataCount);  

The line await docClient.scan(params).promise(); seems to work fine. It does what I expect calling the function and waiting until the response is ready. However, await firehose.putRecordBatch(params).promise(); does not seem to work. The output just prints 'Interventions in DB = ...' completely skipping over the firehose part.

I seem to fight with async functions in node.js lambdas a lot. There doesn't seem to be a standard approach that works reliably.

Updates

I tried using a little function:

async function sendToFirehose(params) {
    return Promise((resolve, reject) => {
        firehose.putRecordBatch(params, (err, data) => {
            if (err) return reject(err);
            return resolve(data);
        });
    });
}

...
await sendToFirehose(params);

But get the error:

{
"errorType": "Runtime.UnhandledPromiseRejection",
"errorMessage": "TypeError: undefined is not a promise",
"reason": {
    "errorType": "TypeError",
    "errorMessage": "undefined is not a promise",
    "stack": [
        "TypeError: undefined is not a promise",
        "    at Promise (<anonymous>)",
        "    at sendToFirehose (/var/task/index.js:76:12)",
        "    at dynamoDBToCSVArray (/var/task/index.js:160:11)",
        "    at /var/task/index.js:56:21",
        "    at Array.forEach (<anonymous>)",
        "    at updateAll (/var/task/index.js:47:21)",
        "    at processTicksAndRejections (internal/process/task_queues.js:97:5)",
        "    at async Runtime.exports.handler (/var/task/index.js:11:19)"
    ]
},
"promise": {},
"stack": [
    "Runtime.UnhandledPromiseRejection: TypeError: undefined is not a promise",
    "    at process.<anonymous> (/var/runtime/index.js:35:15)",
    "    at process.emit (events.js:315:20)",
    "    at process.EventEmitter.emit (domain.js:483:12)",
    "    at processPromiseRejections (internal/process/promises.js:209:33)",
    "    at processTicksAndRejections (internal/process/task_queues.js:98:32)"
]

}

Answer 1

I too have seen many such instances where AWS sdk just messing up with promises. Instead I'd suggest you to wrap the callback in a Promise.

You can do something like this:

async function dynamoDBToCSV(record) {
    return Promise((resolve, reject) => {
        firehose.putRecordBatch(params, (err, data) => {
            if (err) return reject(err);
            return resolve(data);
        });
    }
}