const puppeteer = require('puppeteer'); const axios = require('axios') const path = require('path') const apiKey = "6hUfJdAartHTHhHc0WIRZYPWe" // Spark Production ^ // const apiKey = "p5fidpuedHaOlPnd8EcpxzQMG" // Luminate Production ^ const uid = "/\?uid\=7beg87y4-fh24-4929-3rt5-24kdn87s5241"; const groupIds = [] const getAllGroups = async (num) => { if(num === 1){ } let page = num; await axios({ method: 'get', url: 'https://api2.northpass.com/v2/courses?page=${page}&limit=200', headers: { 'accept': '*/*', 'x-api-key': apiKey, 'content-type': 'application/json' } }) .then(async (res) => { if (res.data.links.next != null) { page++; for (let i = 0; i < res.data.data.length; i++) { if (res.data.data[i].attributes.status == "live") { console.log(res.data.data[i].attributes.name) groupIds.push(res.data.data[i].id) } } // await getAllGroups(page); } else { for (let i = 0; i < res.data.data.length; i++) { if (res.data.data[i].attributes.status == "live") { console.log(res.data.data[i].attributes.name) groupIds.push(res.data.data[i].id) } } } }) .catch(err => { console.log(err); }) let i = 0 loopIds(i) console.log(groupIds) } function loopIds(i) { console.log("Loop Ids Function") var num = 1 id = groupIds[i] courseOverview(id, i, num) } async function courseOverview(id, i, num) { // console.log("Course Overview Function") const activity = new Array(); const url = "https://walmart.northpass.com/app/courses/"; const browser = await puppeteer.launch(); const page = await browser.newPage(); const course = url+id+uid; console.log(course) if (course.includes("undefined")) { console.log("Error - Undefined UUID. Possibly end of list. Exiting.") await browser.close(); } else { await page.setViewport({ width:390, height:844 }) // await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3419.0 Safari/537.36'); await page.goto(course, { waitUntil: 'load', timeout: 0 }); await page.title(); const [getXpath] = await page.$x('/html/body/div[1]/div'); const resourcetitle = await page.evaluate(name => name.innerText, getXpath); console.log(resourcetitle) const resource = resourcetitle.trim(); // console.log(resource); await page.screenshot({path: `${resource}_1.png`, fullPage:true, headless:"new"}); const hrefs = await page.$$eval('a', as => as.map(a => a.href)); var links = Object.entries(hrefs); links.forEach(([key, value]) => { var link = value; if (link.includes("activities") && (link.includes(id))) { console.log("Activities Link") console.log(link); activity.push(link); } }); setTimeout(() => { }, 10000); uniqueLinks = [...new Set(activity)]; for (const link of uniqueLinks) { num = num+1 const newlink = link+uid; console.log("New Link: " + newlink) await page.setViewport({ width: 390, height: 844 }) await page.goto(newlink, { waitUntil: 'load', timeout: 0 }); await page.screenshot({path: `${resource}_${num}.png`, fullPage: true, headless:"new"}); screenshots(newlink, resource, num, i); } await browser.close(); i++; loopIds(i); } } async function screenshots(newlink, resource, num, i) { const browser = await puppeteer.launch(); const page = await browser.newPage(); await browser.close(); } getAllGroups();