sisa-crawl/iterateOverDashboardTable.js

/* TEST MODUS
open chrome in debug modus
start chrome --remote-debugging-port=9222 --user-data-dir="C:\ChromeDebug"
navigeer naar het dashboard en roep de lijst met startpakket studenten op die relevant zijn voor de test
wees er zeker van dat de tab waarin je moet werken de meest rechtse tab is in de tabbladenbalk
start dan het script
 */
const puppeteer = require("puppeteer");
const isStartPakketAvailable = require("./evaluateStartPakket");
const fs = require("fs");
const saveResultsToExcel = require("./saveResultsToExcel");
const appendToExcel = require("./appendResultsToExcel");
const wait = require("./wait");
const courseEnrolled = require("./coursesEnrolled");
const parseCourseResults = require('./utils/parseCourseResults');
const config = require("./config.js");
const updateResultsDatabase = require("./utils/updateResultsDatabase");

async function iterateOverDashboardTable() {
  /*connection to local host */
  const browser = await puppeteer.connect({
    browserURL: "http://localhost:9222"
  });

  //array to store results for Excel export
  let results = [];
  const filename = "DashboardResults.xlsx";

  // Get all open pages (tabs)
  const pages = await browser.pages();
  console.log(`Found ${pages.length} open pages on the browser.`);

  // Interact with the last tab (or any tab of choice)
  let page = pages[pages.length - 1]; // Choose the last opened tab
  console.log("Current URL:", page.url());

  // Select the table body by ID
  const iframeElement = await page.waitForSelector(
      config.iframeSelector
  );
  const iframe = await iframeElement.contentFrame();

  if (!iframe) {
    console.log("Iframe on dasbhoard inschrijvingen page not found!");
    return;
  }

  // Check if the table exists inside the iframe
  const tableExists = await iframe.$(config.tableBodySelector);
  if (!tableExists) {
    console.log("Table not found inside iframe!");
    return;
  }
  console.log("Table dashboard found inside iframe!");

  // Get all rows within the table body
  const links = await iframe.$$(
        `${config.tableBodySelector} tr span[title="AA-rapport"] a`
  );
  console.log(`Found ${links.length} AA-links to process.`);

  // Initialize Excel file if it doesn't exist
  if (!fs.existsSync(filename)) {
    saveResultsToExcel([], filename); // Create empty file
    console.log(`Initialized new Excel file: ${filename}`);
  }

  // process links
  for (let i = 0; i < links.length; i++) {
    console.log(`Processing link ${i + 1}`);

    const iframeElement = await page.waitForSelector(
      config.iframeSelector
    );
    const iframe = await iframeElement.contentFrame();

    const links = await iframe.$$(
        `${config.tableBodySelector} tr span[title="AA-rapport"] a`
    );
    // Get the parent row of the current link
    const parentRow = await links[i].evaluateHandle((link) =>
      link.closest("tr")
    );
    const rowContainsBeëindigd = await parentRow.evaluate((row) =>
      row.innerText.includes("Beëindigd")
    );

    if (rowContainsBeëindigd) {
      console.log(`Skipping link ${i + 1} as the row contains 'Beëindigd'`);
      continue;
    }
    if ((i + 1) % 10 === 0) {
      console.log(
        "Simulating break, waiting 10 seconds to stay under the radar :)"
      );
      await wait(10000);
    }

    const [newPagePromise] = await Promise.all([
      new Promise((resolve) =>
        browser.once("targetcreated", async (target) => {
          const newPage = await target.page();
          resolve(newPage);
        })
      ),
      links[i].click(), // Simulate the click
    ]);
    let result;

    const newPage = await newPagePromise;

    try {
      const evaluationResult = await isStartPakketAvailable(newPage);
      const coursesResult = await courseEnrolled(newPage)

      //Save results for Excel
      result = {
        LinkNumber: i + 1,
        StudentName: evaluationResult.studName || "N/A",
        StudentID: evaluationResult.studId || "N/A",
        IsCollapsed: evaluationResult.isCollapsed ? "Yes" : "No",
        IsCollapsedKeuze: evaluationResult.isCollapsed_keuze ? "Yes" : "No",
        IsCollapsedMain: evaluationResult.isCollapsed_main ? "Yes" : "No",
        ContainsVZP: evaluationResult.containsVZP ? "Yes" : "No",
        ContainsVZPKeuze: evaluationResult.containsVZP_keuze ? "Yes" : "No",
        coursesResult: parseCourseResults(coursesResult.rowData),
        totalStudiepunten: coursesResult.totalStudiepunten
      };
      console.log("Courses enrolled: ", coursesResult.rowData);
      console.log(`Link ${i + 1} processed successfully.`);
    } catch (error) {
      console.error(`Error processing link ${i + 1}:`, error.message);
      // Save error for Excel
      result = {
        LinkNumber: i + 1,
        StudentName: "Error",
        StudentID: "Error",
        IsCollapsed: "Error",
        IsCollapsedKeuze: "Error",
        IsCollapsedMain: "Error",
        ContainsVZP: "Error",
        ContainsVZPKeuze: "Error",
        ErrorStr: error.message,
      };
    } finally {
      await newPage.close();
    }
    results.push(result);
    appendToExcel(result, filename);
    updateResultsDatabase(result)
  }


  console.log("All links processed.");
  saveResultsToExcel(results, config.excelFilename);
  console.log("Results saved successfully to DashboardResults.xlsx");
}

module.exports = iterateOverDashboardTable();