Handling callbacks with multiple nested functions and for loops?

Question

I'm about a week into learning the javascript syntax and had no idea about the whole asynchronous - synchronous situation. I created a function that scrapes names from a page, adds those names to a checklist array, searches each name in the checklist individually and then scrapes the names found in the resulting pages, and so on, until no more names could be found. Kinda like a micro web crawler.

Anyway that context is beside the issue. My goal is to add the resulting array after all functions and for loops are executed to a txt file. I'm only doing this to make the debugging process a little easier to manage so what happens afterwards doesn't really matter. I'm just trying to understand how to execute these functions in a manner that mimics synchronous processing. I've looked into callbacks, async/await functions, and promises with no luck at successfully implementing them in this script. I've got many nested functions and for loops so my guess is that i'm running into scoping issues.

This script is a mock up of what i'm actually scraping. However the actual script is the same, just changed the variable names and class names being scraped. For the issue at hand, i'm sure this won't matter.

const fetch = require('node-fetch');
const cheerio = require('cheerio');

const fruit_names_checklist = []
const fruit_names = [];

fruitsearch = (callback) => {

    // defines url and search term

    const url = "https://www.google.com/search?q=types+of+"
    let fruit_name = "banana"

    // gets html of page

    findfruits(fruit_name);
    function findfruits(thefruit) {
        return fetch(`${url}${thefruit}`)
        .then(response => response.text())
        .then(body => {

            // scrapes fruit names and moves them into fruit_names array

            const $ = cheerio.load(body);
            $('.llgymd').each( (index, element) => {
                const $element = $(element);
                const names = $element.text();
                fruit_names[index] = names;
            });

            /* for loop that adds each fruit name to fruit_names_checklist 
               if it doesn't already doesn't already exist in the checklist */

            for(j = 0; j < fruit_names.length; j++){
                if(!fruit_names_checklist.includes(fruit_names[j])){
                     fruit_names_checklist.push(fruit_names[j]);
                     console.log(fruit_names_checklist);
            }}
 
                /* for loop that goes through each name in the checklist
                   and now searches each name through the same url request. */

                for(let i = 0; i < fruit_names_checklist.length; i++){

                    let fruit_name = fruit_names_checklist[i]
                    findfruits_loop(fruit_name);

                    function findfruits_loop(thefruit) {
                        return fetch(`${url}${thefruit}`)
                        .then(response => response.text())
                        .then(body => {
                            const $ = cheerio.load(body);
                            $('.llgymd').each( (index, element) => {
                                const $element = $(element);
                                const names = $element.text();
                                fruit_names[index] = names;
                            });

                            /* Again adds each fruit name found to fruit_names_checklist 
                               if it doesn't already exist in the checklist */

                            for(j = 0; j < fruit_names.length; j++){
                                if(!fruit_names_checklist.includes(fruit_names[j])){
                                    fruit_names_checklist.push(fruit_names[j]);
                                }
                                console.log(fruit_names_checklist);
                            }
                            // where i've placed the callback
                            callback()
                        });
                    }
                }
            })
        }
    }

    fruitsearch(
    array_to_text = () => {
        // library that converts array to txt file
        const arrayToTxtFile = require('array-to-txt-file')
        const randomValue_6d = Math.floor(Math.random() * 900000)
        arrayToTxtFile(fruit_names_checklist, `backend_modules/debugging/arrays_to_txt/fruit_names_checklist-${randomValue_6d}.txt`, err => {
            if(err) {
                console.error(err)
                return }
                console.log('Array successfully wrote to txt file')});
            })

I console logged each fruit name in the for loop that successfully added to the fruit_names_checklist. As you see at the end in the console.log list below, it successfully created a txt file after all loops and functions were completed, however it was called inside of the "i" for loop. So of course created a new txt file for each iteration, which definitely isn't desirable.

Array(1) ["Plantain"]
Array(2) ["Plantain", "Hardy banana"]
Array(3) ["Plantain", "Hardy banana", "Blue Java banana"]
Array(4) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana"]
Array(5) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata"]
Array(6) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana"]
Array(7) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana"]
Array(8) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina"]
Array(9) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(10) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(11) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(12) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(13) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(14) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(15) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(16) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(17) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(18) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(19) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(20) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(21) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(22) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(23) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(24) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(25) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(26) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(27) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(28) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(29) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(30) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(31) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(32) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(33) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(34) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(35) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(36) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(37) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(38) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(39) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Canceled
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file
Array successfully wrote to txt file

This is the closest I've gotten to creating the txt file after all functions and loops are run, if I use callback() at the end of the entire "fruitsearch" function, the txt file is created before any data is added to the checklist. I've tried to callback in almost every place in this main function, either it creates the txt file before any functions are executed or it creates it in a for loop.

This is the desired outcome

Array(1) ["Plantain"]
Array(2) ["Plantain", "Hardy banana"]
Array(3) ["Plantain", "Hardy banana", "Blue Java banana"]
Array(4) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana"]
Array(5) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata"]
Array(6) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana"]
Array(7) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana"]
Array(8) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina"]
Array(9) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(10) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(11) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(12) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(13) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(14) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(15) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(16) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(17) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(18) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(19) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(20) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(21) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(22) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(23) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(24) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(25) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(26) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(27) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(28) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(29) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(30) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(31) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(32) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(33) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(34) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(35) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(36) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(37) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(38) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Array(39) ["Plantain", "Hardy banana", "Blue Java banana", "Cavendish banana", "Musa acuminata", "Musa balbisiana", "Red banana", "Musa velutina", …]
Canceled
Array successfully wrote to txt file

I can't wrap my head around using "callback()" in the context of many nested functions and many nested for loops. Seems that my scoping is off in some way. Looking forward to see what I'm doing wrong here.

Thanks in advance

There's so much noise in this question it's making my eye twitch. Try to keep things concise, boil it down to the root problem, don't paste in reams of code/logs. Simply saying "My callback gets called more than once" would have sufficed. As for learning JS - callbacks are straight-forward enough, understanding Promises and Async/Await is a must and I suggest you take the time to read up and understand them properly. Without actually answering my only feedback about the code is: make lots of functions. Break the problem down. After a while you'll see the power of chaining functions. — Richard Dunn
– Richard Dunn, Commented Apr 30, 2020 at 19:18

marc_s · Accepted Answer · 2020-05-07 12:14:50Z

I don't really know if this will help, but when facing situation like yours I use a counter like this.

fruitsearch = (callback) => {
  // defines url and search term

  const url = "https://www.google.com/search?q=types+of+";
  let fruit_name = "banana";

  // gets html of page

  findfruits(fruit_name);
  function findfruits(thefruit) {
    return fetch(`${url}${thefruit}`)
      .then((response) => response.text())
      .then((body) => {
        // scrapes fruit names and moves them into fruit_names array

        const $ = cheerio.load(body);
        $(".llgymd").each((index, element) => {
          const $element = $(element);
          const names = $element.text();
          fruit_names[index] = names;
        });

        /* for loop that adds each fruit name to fruit_names_checklist 
           if it doesn't already doesn't already exist in the checklist */

        for (j = 0; j < fruit_names.length; j++) {
          if (!fruit_names_checklist.includes(fruit_names[j])) {
            fruit_names_checklist.push(fruit_names[j]);
            console.log(fruit_names_checklist);
          }
        }

        /* for loop that goes through each name in the checklist
           and now searches each name through the same url request. */
        let count = fruit_names_checklist.length; // initialize the counter
        for (let i = 0; i < fruit_names_checklist.length; i++) {
          count--; // decrement 
          let fruit_name = fruit_names_checklist[i];
          findfruits_loop(fruit_name);

          function findfruits_loop(thefruit) {
            return fetch(`${url}${thefruit}`)
              .then((response) => response.text())
              .then((body) => {
                const $ = cheerio.load(body);
                $(".llgymd").each((index, element) => {
                  const $element = $(element);
                  const names = $element.text();
                  fruit_names[index] = names;
                });

                /* Again adds each fruit name found to fruit_names_checklist 
                   if it doesn't already exist in the checklist */

                for (j = 0; j < fruit_names.length; j++) {
                  if (!fruit_names_checklist.includes(fruit_names[j])) {
                    fruit_names_checklist.push(fruit_names[j]);
                  }
                  console.log(fruit_names_checklist);
                }
                // where I've placed the callback
                if (count == 0) callback(); // if all iteration are passed write the txt file
              });
          }
        }
      });
  }
};

Collectives™ on Stack Overflow

Handling callbacks with multiple nested functions and for loops?

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related