bdaneels/sisa-crawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor evaluateStartPakket.js and iterateOverDashboardTable.js

Browse Source
This commit is contained in:
bdaneels 2024-11-22 14:47:09 +01:00
parent a554020123
commit b92f1c8c29
2 changed files with 104 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
evaluateStartPakket.js
View File

@ -1,6 +1,6 @@
const IFRAME_SELECTOR = 'iframe[title="Hoofdinhoud"]'; const IFRAME_SELECTOR = 'iframe[title="Hoofdinhoud"]';
const CELL_SELECTOR = 'a[title="Sectie uitvouwen Algemene opleidingsonderdelen"]'; const CELL_SELECTOR =
'a[title="Sectie uitvouwen Algemene opleidingsonderdelen"]';
/** /**
* Extracted function to get the text content of a node. * Extracted function to get the text content of a node.
@ -8,7 +8,7 @@ const CELL_SELECTOR = 'a[title="Sectie uitvouwen Algemene opleidingsonderdelen"]
* @returns {Promise<string>} The text content of the node. * @returns {Promise<string>} The text content of the node.
*/ */
async function getNodeTextContent(node) { async function getNodeTextContent(node) {
return await node.evaluate(node => node.textContent); return await node.evaluate((node) => node.textContent);
} }
/** /**
@ -17,13 +17,13 @@ async function getNodeTextContent(node) {
* @returns {Promise<Frame>} The content frame. * @returns {Promise<Frame>} The content frame.
*/ */
async function getContentFrame(page) { async function getContentFrame(page) {
await page.waitForSelector(IFRAME_SELECTOR); await page.waitForSelector(IFRAME_SELECTOR);
const iframe = await page.$(IFRAME_SELECTOR); const iframe = await page.$(IFRAME_SELECTOR);
if (!iframe) { if (!iframe) {
throw new Error('Could not find iframe on the AA-page'); throw new Error("Could not find iframe on the AA-page");
} }
console.log('iframe found on AA-page') console.log("iframe found on AA-page");
return await iframe.contentFrame(); return await iframe.contentFrame();
} }
/** /**
@ -34,21 +34,24 @@ async function getContentFrame(page) {
* @returns {Promise<boolean>} True if the 'Startpakket' is available, false otherwise. * @returns {Promise<boolean>} True if the 'Startpakket' is available, false otherwise.
*/ */
async function isStartPakketAvailable(page) { async function isStartPakketAvailable(page) {
const contentFrame = await getContentFrame(page); const contentFrame = await getContentFrame(page);
await contentFrame.waitForSelector(CELL_SELECTOR); await contentFrame.waitForSelector(CELL_SELECTOR);
const cell = await contentFrame.$(CELL_SELECTOR); const cell = await contentFrame.$(CELL_SELECTOR);
console.log(cell);
if (!cell) { if (!cell) {
console.error('Cell not found'); throw new Error(
return false; "Cell inside function isStartPakketAvailable is not defined"
} );
}
const isCollapsed = await isAriaExpandedFalse(cell); const isCollapsed = await isAriaExpandedFalse(cell);
console.log(`Aria-expanded: ${isCollapsed}`); console.log(`Aria-expanded: ${isCollapsed}`);
return isCollapsed; return {
isCollapsed: isCollapsed,
// Add more properties here in the future as needed
};
} }
/** /**
@ -58,10 +61,12 @@ async function isStartPakketAvailable(page) {
* @returns {Promise<boolean>} True if the `aria-expanded` attribute is 'false', false otherwise. * @returns {Promise<boolean>} True if the `aria-expanded` attribute is 'false', false otherwise.
*/ */
async function isAriaExpandedFalse(node) { async function isAriaExpandedFalse(node) {
if (!node) { if (!node) {
throw new Error('Node is not defined'); throw new Error("Node is not defined");
} }
return await node.evaluate(element => element.getAttribute('aria-expanded') === 'false'); return await node.evaluate(
(element) => element.getAttribute("aria-expanded") === "false"
);
} }
module.exports = isStartPakketAvailable ; module.exports = isStartPakketAvailable;

143
iterateOverDashboardTable.js
View File

@ -8,83 +8,86 @@ const puppeteer = require("puppeteer");
const isStartPakketAvailable = require("./evaluateStartPakket"); const isStartPakketAvailable = require("./evaluateStartPakket");
async function iterateOverDashboardTable() { async function iterateOverDashboardTable() {
/*connection to local host */
const browser = await puppeteer.connect({
browserURL: "http://localhost:9222", // Connect to the browser's debugging address
});
/*connection to local host */ // Get all open pages (tabs)
const browser = await puppeteer.connect({ const pages = await browser.pages();
browserURL: 'http://localhost:9222', // Connect to the browser's debugging address console.log(`Found ${pages.length} open pages on the browser.`);
});
// Get all open pages (tabs) // Interact with the last tab (or any tab of choice)
const pages = await browser.pages(); let page = pages[pages.length - 1]; // Choose the last opened tab
console.log(`Found ${pages.length} open pages on the browser.`); console.log("Current URL:", page.url());
// Interact with the last tab (or any tab of choice) // Select the table body by ID
let page = pages[pages.length - 1]; // Choose the last opened tab const iframeElement = await page.waitForSelector(
console.log('Current URL:', page.url()); 'iframe[title="Hoofdinhoud"]'
);
const iframe = await iframeElement.contentFrame();
// Select the table body by ID if (!iframe) {
const iframeElement = await page.waitForSelector('iframe[title="Hoofdinhoud"]'); console.log("Iframe on dasbhoard inschrijvingen page not found!");
return;
}
// Select the table body within the iframe
const tableBodySelector = "#QQ_RPT_OIS_TEMP\\$scroll\\$0 tbody";
// Check if the table exists inside the iframe
const tableExists = await iframe.$(tableBodySelector);
if (!tableExists) {
console.log("Table not found inside iframe!");
return;
}
console.log("Table dashboard found inside iframe!");
// Get all rows within the table body
const links = await iframe.$$(
`${tableBodySelector} tr span[title="AA-rapport"] a`
);
console.log(`Found ${links.length} AA-links to process.`);
for (let i = 0; i < links.length; i++) {
console.log(`Processing link ${i + 1}`);
const iframeElement = await page.waitForSelector(
'iframe[title="Hoofdinhoud"]'
);
const iframe = await iframeElement.contentFrame(); const iframe = await iframeElement.contentFrame();
if (!iframe) { const links = await iframe.$$(
console.log('Iframe on dasbhoard inschrijvingen page not found!'); `${tableBodySelector} tr span[title="AA-rapport"] a`
return; );
const [newPagePromise] = await Promise.all([
new Promise((resolve) =>
browser.once("targetcreated", async (target) => {
const newPage = await target.page();
resolve(newPage);
})
),
links[i].click(), // Simulate the click
]);
const newPage = await newPagePromise;
try {
const evaluationResult = await isStartPakketAvailable(newPage);
if (evaluationResult.isCollapsed) {
console.log(`Evaluation succeeded for link ${i + 1}`);
} else {
console.log(`Evaluation failed for link ${i + 1}`);
}
} catch (error) {
console.error(`Error processing link ${i + 1}:`, error.message);
} }
// Select the table body within the iframe await newPage.close();
const tableBodySelector = '#QQ_RPT_OIS_TEMP\\$scroll\\$0 tbody'; }
// Check if the table exists inside the iframe
const tableExists = await iframe.$(tableBodySelector);
if (!tableExists) {
console.log('Table not found inside iframe!');
return;
}
console.log('Table dashboard found inside iframe!');
// Get all rows within the table body
const links = await iframe.$$(`${tableBodySelector} tr span[title="AA-rapport"] a`);
console.log(`Found ${links.length} AA-links to process.`);
for (let i = 0; i < links.length; i++) {
console.log(`Processing link ${i + 1}`);
const iframeElement = await page.waitForSelector('iframe[title="Hoofdinhoud"]');
const iframe = await iframeElement.contentFrame();
const links = await iframe.$$(`${tableBodySelector} tr span[title="AA-rapport"] a`);
const [newPagePromise] = await Promise.all([
new Promise((resolve) =>
browser.once('targetcreated', async (target) => {
const newPage = await target.page();
resolve(newPage);
})
),
links[i].click(), // Simulate the click
]);
const newPage = await newPagePromise;
try {
const evaluationResult = await isStartPakketAvailable(newPage);
if (evaluationResult) {
console.log(`Evaluation succeeded for link ${i + 1}`);
} else {
console.log(`Evaluation failed for link ${i + 1}`);
}
} catch (error) {
console.error(`Error processing link ${i + 1}:`, error.message);
}
await newPage.close();
}
console.log('All links processed.');
console.log("All links processed.");
} }
module.exports = iterateOverDashboardTable(); module.exports = iterateOverDashboardTable();