norm/Gainsight
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Anthology script for domain names. Used get courses for Walmart, and made a backup of the screenshots for 1-off courses. Glassdoor and todo notes.

Browse Source
This commit is contained in:
Norm Rasmussen
2024-03-11 17:03:07 -04:00
parent a3455a720c
commit 6576919ff6
6 changed files with 126 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
Scripts/Auto_Scrape_Screenshots/resources_api.js
View File

@ -17,7 +17,7 @@ const getAllGroups = async (num) => {
await axios({
method: 'get',
url: 'https://api2.northpass.com/v2/courses?page=${page}&limit=200',
url: 'https://api2.northpass.com/v2/courses?page=${page}&limit=100',
headers: {
'accept': '*/*',
'x-api-key': apiKey,
@ -83,17 +83,17 @@ async function courseOverview(id, i, num) {
const resource = resourcetitle.trim();
console.log(resource);
await page.screenshot({path: `${resource}_1.png`, fullPage:true, headless:"new"});
await page.screenshot({path: `${resource}_1.png`, fullPage:true });
const hrefs = await page.$$eval('a', as => as.map(a => a.href));
var links = Object.entries(hrefs);
console.log(links)
// console.log(links)
links.forEach(([key, value]) => {
var link = value;
if (link.includes("activities") && (link.includes(id))) {
console.log("Activities Link")
console.log(link);
activity.push(link);
var link = value;
if (link.includes("activities") && (link.includes(id))) {
// console.log("Activities Link")
// console.log(link);
activity.push(link);
}
});
setTimeout(() => {
@ -109,7 +109,7 @@ async function courseOverview(id, i, num) {
await page.goto(newlink, {
waitUntil: 'load',
timeout: 0 });
await page.screenshot({path: `${resource}_${num}.png`, fullPage: true, headless:"new"});
await page.screenshot({path: `${resource}_${num}.png`, fullPage: true });
screenshots(newlink, resource, num, i);
}
await browser.close();