Using Promises with fs.readFile in a loop

Question

I'm trying to understand why the below promise setups don't work.

(Note: I already solved this issue with async.map. But I would like to learn why my attempts below didn't work.)

The correct behavior should be: bFunc should run as many time as necessary to fs read all the image files (bFunc below runs twice) and then cFunc console prints "End".

Thanks!

Attempt 1: It runs and stops at cFunc().

var fs = require('fs');

bFunc(0)
.then(function(){ cFunc() }) //cFunc() doesn't run

function bFunc(i){
    return new Promise(function(resolve,reject){

        var imgPath = __dirname + "/image1" + i + ".png";

        fs.readFile(imgPath, function(err, imagebuffer){

            if (err) throw err;
            console.log(i)

            if (i<1) {
                i++;
                return bFunc(i);
            } else {
                resolve();
            };

        });

    })
}

function cFunc(){
    console.log("End");
}

Attempt 2: In this case, I used a for-loop but it executes out of order. Console prints: End, bFunc done, bFunc done

var fs = require('fs');

bFunc()
        .then(function(){ cFunc() })

function bFunc(){
    return new Promise(function(resolve,reject){

        function read(filepath) {
            fs.readFile(filepath, function(err, imagebuffer){
                if (err) throw err;
                console.log("bFunc done")
            });
        }

        for (var i=0; i<2; i++){
            var imgPath = __dirname + "/image1" + i + ".png";
            read(imgPath);
        };

        resolve()
    });
}


function cFunc(){
    console.log("End");
}

Thanks for the help in advance!

In attempt 1, where is you error handler for bFunc? If an error is thrown, you will never know about it with your current code. — nils
– nils, Commented Jan 6, 2016 at 8:23
What is the goal of this code? Please describe the problem you are trying to solve in words rather than just showing us code that doesn't do what you want. There are lots of things wrong with your code so I'd rather start from understand the problem to be solved than trying to rework all the things wrong with your code without knowing the actual end goal. — jfriend00
– jfriend00, Commented Jan 6, 2016 at 8:35
@nils it doesn't throw an error at bFunc! But I guess I should put a catch at the end of the chain. — David
– David, Commented Jan 6, 2016 at 17:48
@jfriend00 I stripped out all the details of the code and just left the structure to simplify it. The problem is the same but if it helps, I was using Microsoft's Project Oxford for facial detection and then in each ".then" I augment the photo with glasses, hats, etc. I can share the source code with you. Thanks. — David
– David, Commented Jan 6, 2016 at 17:57
Well, think about it from our perspective. You give us two blocks of code that don't work and ask us how to fix them. But, you never really describe what they're supposed to do (what the correct behavior is). Are we supposed to guess from the malfunctioning code what the proper behavior is? Please describe the desired and proper behavior you want out of either code block. — jfriend00
– jfriend00, Commented Jan 6, 2016 at 18:12

jfriend00 · Accepted Answer · 2019-07-25 21:13:02Z

106

So, anytime you have multiple async operations to coordinate in some way, I immediately want to go to promises. And, the best way to use promises to coordinate a number of async operations is to make each async operation return a promise. The lowest level async operation you show is fs.readFile(). Since I use the Bluebird promise library, it has a function for "promisifying" a whole module's worth of async functions.

var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));

This will create new parallel methods on the fs object with an "Async" suffix that return promises instead of use straight callbacks. So, there will be an fs.readFileAsync() that returns a promise. You can read more about Bluebird's promisification here.

So, now you can make a function that gets an image fairly simply and returns a promise whose value is the data from the image:

 function getImage(index) {
     var imgPath = __dirname + "/image1" + index + ".png";
     return fs.readFileAsync(imgPath);
 }

Then, in your code, it looks like you want to make bFunc() be a function that reads three of these images and calls cFunc() when they are done. You can do that like this:

var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));

 function getImage(index) {
     var imgPath = __dirname + "/image1" + index + ".png";
     return fs.readFileAsync(imgPath);
 }

 function getAllImages() {
    var promises = [];
    // load all images in parallel
    for (var i = 0; i <= 2; i++) {
        promises.push(getImage(i));
    }
    // return promise that is resolved when all images are done loading
    return Promise.all(promises);
 }

 getAllImages().then(function(imageArray) {
    // you have an array of image data in imageArray
 }, function(err) {
    // an error occurred
 });

If you did not want to use Bluebird, you could manually make a promise version of fs.readFile() like this:

// make promise version of fs.readFile()
fs.readFileAsync = function(filename) {
    return new Promise(function(resolve, reject) {
        fs.readFile(filename, function(err, data){
            if (err) 
                reject(err); 
            else 
                resolve(data);
        });
    });
};

Or, in modern versions of node.js, you can use util.promisify() to make a promisified version of a function that follows the node.js async calling convention:

const util = require('util');
fs.readFileAsync = util.promisify(fs.readFile);

Though, you will quickly find that once you start using promises, you want to use them for all async operations so you'll be "promisifying" lots of things and having a library or at least a generic function that will do that for you will save lots of time.

In even newer versions of node.js (version 10.0+), you can use the built-in version of the fs library that supports promises:

const fsp = require('fs').promises;

fsp.readFile("someFile").then(data => {
    console.log(data);
});

edited Jul 25, 2019 at 21:13

answered Jan 6, 2016 at 21:12

jfriend00

711k104 gold badges1.1k silver badges1k bronze badges

Sign up to request clarification or add additional context in comments.

14 Comments

Jules Sam. Randolph Over a year ago

please note that you now have to use Promise.promisifyAll instead of Promise.promisify

Daniel Over a year ago

Note that node 8 comes with it's own promisify function: const {promisify} = require("util"); const fs = require("fs"); const readFile = promisify(fs.readFile);

jfriend00 Over a year ago

@Daniel - Using util.promisify() as an option in newer versions of node.js is already in my answer.

jfriend00 Over a year ago

@MzA - The standard way now would be the last code block using fs.promises.readFile. I'll look at reorganizing the answer.

jfriend00 Over a year ago

@AlexMelgar - Note that the answer also refers to the more modern const fsp = require('fs').promises; so you can just use the built-in promisified versions and there really isn't much of a use for Bluebird.promisifyAll() any more (at least not for the fs module).

|

Dmitry Yudakov · Accepted Answer · 2019-08-16 15:11:48Z

78

Node v10 has fs Promises API

const fsPromises = require('fs').promises

const func = async filenames => {

  for(let fn of filenames) {
    let data = await fsPromises.readFile(fn)
  }

}

func(['file1','file2'])
  .then(res => console.log('all read', res))
  .catch(console.log)

https://nodejs.org/api/fs.html#fs_fs_promises_api

Or if you want to read more files simultaneously:

const func = filenames => {
  return Promise.all(
    filenames.map(f => fsPromises.readFile(f))
  )
}

func(['./a','./b'])
  .then(res => console.log('all read', res))
  .catch(console.log)

edited Aug 16, 2019 at 15:11

answered Aug 28, 2018 at 14:52

Dmitry Yudakov

15.8k4 gold badges53 silver badges53 bronze badges

4 Comments

Jaymes Bearden Over a year ago

Note: as of 2019-06-28, this feature has been stable for sometime.

estani Over a year ago

great info! Though your example makes little sense since the files are read in sync, right? Every promise is waited to be resolved before the next one is triggered.

Dmitry Yudakov Over a year ago

@estani This way it's in a loop ;) But here's another example with parallel execution

weiya ou Over a year ago

import { readFile, writeFile } from 'fs/promises'; more modern

Tomalak · Accepted Answer · 2019-01-22 13:06:29Z

26

Your code should look more like this:

// promisify fs.readFile()
fs.readFileAsync = function (filename) {
    return new Promise((resolve, reject) => {
        fs.readFile(filename, (err, buffer) => {
            if (err) reject(err); else resolve(buffer);
        });
    });
};

const IMG_PATH = "foo";

// utility function
function getImageByIdAsync(i) {
    return fs.readFileAsync(IMG_PATH + "/image1" + i + ".png");
}

Usage with a single image:

getImageByIdAsync(0).then(imgBuffer => {
    console.log(imgBuffer);
}).catch(err => {
    console.error(err);
});

Usage with multiple images:

var images = [1,2,3,4].map(getImageByIdAsync);

Promise.all(images).then(imgBuffers => {
    // all images have loaded
}).catch(err => {
    console.error(err);
});

To promisify a function means to take an asynchronous function with callback semantics and derive from it a new function with promise semantics.

It can be done manually, like shown above, or – preferably – automatically. Among others, the Bluebird promise library has a helper for that, see http://bluebirdjs.com/docs/api/promisification.html

edited Jan 22, 2019 at 13:06

answered Jan 6, 2016 at 18:45

Tomalak

340k68 gold badges547 silver badges635 bronze badges

2 Comments

ngnguyen Over a year ago

i just curious why you put the function fs.readFile() in try catch? Callback can handle the error, right?

Andrej Gaspar Over a year ago

nice! thank you. ` return function () { return new Promise(function(resolve, reject) { try { fs.readFile ` that worked for me with gulp4

Jesus is Lord · Accepted Answer · 2021-12-22 23:56:52Z

If you're using .mjs ECMAScript Modules import syntax then here's code that reads a file, based on this GitHub gist comment reply:

import { promises as fs } from 'fs';
let json = await fs.readFile('./package.json', 'utf-8');
console.log(json);

Here's code that does multiple files, based on this answer:

import { promises as fs } from 'fs';

const sequentially = async filenames => {
    for(let fn of filenames) {
        let json = await fs.readFile(fn, 'utf-8');
        console.log(json);
    }
}

const parallel = filenames => {
    return Promise.all(
      filenames.map(fn => fs.readFile(fn, 'utf-8'))
    )
  }

const fns = ['package.json', 'package-lock.json'];
await sequentially(fns);
await parallel(fns);
console.log('all read');

weiya ou · Accepted Answer · 2023-01-31 10:51:27Z

6

more modern

import { readFile, writeFile } from 'fs/promises';

answered Jan 31, 2023 at 10:51

weiya ou

4,7341 gold badge21 silver badges26 bronze badges

Comments

Stijn de Witt · Accepted Answer · 2022-04-23 21:58:53Z

I once wrote a library for using promises in a loop. It's called for-async. It's pretty small but getting all the details right is hard, so you can have a look at this for inspiration maybe.

This mainly takes care of running the promise-returning functions in sequence, waiting for the promise to resolve before proceeding to the next item and of handling errors. This differs from Promise.map because that function runs the promise-returning functions in parallel, which may or may not be what you want.

// example: looping over promise-returning functions with forAsync
forAsync(
  ['a.txt', 'b.txt', 'c.txt'], 
  readFile
)

// worker function that returns a promise
function readFile(name, idx) {
  return new Promise(function(resolve, reject) {
    setTimeout(function(){
      console.info('read file ' + idx + ': ' + name)
      resolve()
    }, 1000)
  })
}

// forAsync
function forAsync(arr, work) {
  function loop(arr, i) {
    return new Promise(function (resolve, reject) {
      if (i >= arr.length) {
        resolve();
      } else try {
        Promise.resolve(work(arr[i], i)).then(function () {
          return resolve(loop(arr, i + 1));
        }).catch(reject);
      } catch (error) {
        reject(error);
      }
    });
  }
  return loop(arr, 0);
}

Changyuan Chen · Accepted Answer · 2019-03-05 04:55:25Z

0

you can also use this module: 'fs-readfile-promise'

var readFile = require('fs-readfile-promise');
readFile(__dirname + '/file1.txt','utf-8').then(function (data){
    console.log("file's name:", data)
    return readFile(__dirname +'/'+data, 'utf-8')
}).then(function (data1){
    console.log('Content data:', data1)
}).catch( function (err){
    console.log(err)
})

answered Mar 5, 2019 at 4:55

Changyuan Chen

3181 gold badge3 silver badges10 bronze badges

Collectives™ on Stack Overflow

Using Promises with fs.readFile in a loop

7 Answers 7

14 Comments

4 Comments

2 Comments

Comments

Comments

Comments

Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

7 Answers 7

14 Comments

4 Comments

2 Comments

Comments

Comments

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related