From 6576919ff6c82b04e732cd351c393b5a8a8b8415 Mon Sep 17 00:00:00 2001 From: Norm Rasmussen Date: Mon, 11 Mar 2024 17:03:07 -0400 Subject: [PATCH] Anthology script for domain names. Used get courses for Walmart, and made a backup of the screenshots for 1-off courses. Glassdoor and todo notes. --- CustomerNotes/Glassdoor/Glassdoor.md | 6 ++ Scripts/API_Tests/get_course_ids.py | 12 +-- Scripts/Anthology/find_domain_names.py | 13 ++- Scripts/Auto_Scrape_Screenshots/.DS_Store | Bin 6148 -> 6148 bytes .../Auto_Scrape_Screenshots/resources_api.js | 18 ++-- .../single_resource.js | 95 ++++++++++++++++++ 6 files changed, 126 insertions(+), 18 deletions(-) create mode 100644 Scripts/Auto_Scrape_Screenshots/single_resource.js diff --git a/CustomerNotes/Glassdoor/Glassdoor.md b/CustomerNotes/Glassdoor/Glassdoor.md index b7f0446f..42246043 100644 --- a/CustomerNotes/Glassdoor/Glassdoor.md +++ b/CustomerNotes/Glassdoor/Glassdoor.md @@ -133,3 +133,9 @@ Remaining Questions: * How to embed PX into their front end templates? * They measure traffic on help center - with PX? * What are they searching for? + +## 03/11/2024 + +* Senior leadership is fully bought in on GD University being at the forefront of customer education. +* Unsure how to leverage more resources +* Need guidance on how the academy can be used at the right moment for strategic conversations between CSM and client. diff --git a/Scripts/API_Tests/get_course_ids.py b/Scripts/API_Tests/get_course_ids.py index 32a469c6..cfe327b2 100644 --- a/Scripts/API_Tests/get_course_ids.py +++ b/Scripts/API_Tests/get_course_ids.py @@ -4,7 +4,7 @@ import pprint import csv pp = pprint.PrettyPrinter(indent=4) -APIKEY = Apikeys.CIN7 +APIKEY = Apikeys.WALMARTPROD HEADERS = { "accept": "application/json", "X-Api-Key": APIKEY, @@ -22,7 +22,7 @@ def get_courses(): # for course_name in COURSES: count += 1 # url = f"https://api.northpass.com/v2/courses/?filter[name][eq]={course_name}" - url = f"https://api.northpass.com/v2/courses/?limit=100&page={count}" + url = f"https://api2.northpass.com/v2/courses/?limit=100&page={count}" response = requests.get(url, headers=HEADERS) response = response.json() nextlink = response["links"] @@ -31,9 +31,9 @@ def get_courses(): status = item['attributes']['status'] if status == 'live': name = item['attributes']['name'] - if "[Core]" in name: - idict = {item["attributes"]["name"]: item["id"]} - list_of_ids.append(idict) + idict = {item["attributes"]["name"]: item["id"]} + list_of_ids.append(idict) + # if "[Core]" in name: if "next" not in nextlink: break @@ -41,7 +41,7 @@ def get_courses(): pp.pprint(list_of_ids) # print(len(list_of_ids)) with open( - "/Users/normrasmussen/Downloads/Cin7-Courses.csv", "a+", newline='\n' + "/Users/normrasmussen/Downloads/Walmart-Live-Courses.csv", "a+", newline='\n' ) as csvfile: for group in list_of_ids: for key, value in group.items(): diff --git a/Scripts/Anthology/find_domain_names.py b/Scripts/Anthology/find_domain_names.py index dd325d3b..8cd04966 100644 --- a/Scripts/Anthology/find_domain_names.py +++ b/Scripts/Anthology/find_domain_names.py @@ -5,6 +5,13 @@ for domain in DOMAINS: URL = f"https://{domain}" session = HTMLSession() resp = session.get(URL) - title = resp.html.find('head > title', first=True) - group_name = title.text.split("|")[0] - print(group_name) + try: + title = resp.html.find('head > title', first=True) + group_name = title.text.split("|") + print(group_name) + except AttributeError as e: + print(e) + finally: + title = resp.html.find('head > meta:nth-child(3)', first=True) + print(title.text) + diff --git a/Scripts/Auto_Scrape_Screenshots/.DS_Store b/Scripts/Auto_Scrape_Screenshots/.DS_Store index ce2df6924669252e689a189c0327c5dcacfa9682..95dcd0dc5eb8e2f363dc0b17a7ccbca0f1d9bd3b 100644 GIT binary patch delta 108 zcmZoMXfc=|#>CJ*F;Q%yo}wTN0|Nsi1A_oVPP$=ma(-^X#6)dImdT2YQcMgClWiF# z*vf8PQ`>lcax|k9j9JYnvw1mV0^8&bj7vAObMSKjjoZAC@jLTmei1`PhRHTO(wk#M HRxkqqPOcwg delta 193 zcmZoMXfc=|#>B`mF;Q%yo}wTt0|Nsi1A_oVN^x>dQht8UWJbo-Vjxi#h5!Z^1~-Oc zm<+?j9;sLf7G$a1d>5Cboctu97zYsV4`)5 { await axios({ method: 'get', - url: 'https://api2.northpass.com/v2/courses?page=${page}&limit=200', + url: 'https://api2.northpass.com/v2/courses?page=${page}&limit=100', headers: { 'accept': '*/*', 'x-api-key': apiKey, @@ -83,17 +83,17 @@ async function courseOverview(id, i, num) { const resource = resourcetitle.trim(); console.log(resource); - await page.screenshot({path: `${resource}_1.png`, fullPage:true, headless:"new"}); + await page.screenshot({path: `${resource}_1.png`, fullPage:true }); const hrefs = await page.$$eval('a', as => as.map(a => a.href)); var links = Object.entries(hrefs); - console.log(links) + // console.log(links) links.forEach(([key, value]) => { - var link = value; - if (link.includes("activities") && (link.includes(id))) { - console.log("Activities Link") - console.log(link); - activity.push(link); + var link = value; + if (link.includes("activities") && (link.includes(id))) { + // console.log("Activities Link") + // console.log(link); + activity.push(link); } }); setTimeout(() => { @@ -109,7 +109,7 @@ async function courseOverview(id, i, num) { await page.goto(newlink, { waitUntil: 'load', timeout: 0 }); - await page.screenshot({path: `${resource}_${num}.png`, fullPage: true, headless:"new"}); + await page.screenshot({path: `${resource}_${num}.png`, fullPage: true }); screenshots(newlink, resource, num, i); } await browser.close(); diff --git a/Scripts/Auto_Scrape_Screenshots/single_resource.js b/Scripts/Auto_Scrape_Screenshots/single_resource.js new file mode 100644 index 00000000..ac2b8543 --- /dev/null +++ b/Scripts/Auto_Scrape_Screenshots/single_resource.js @@ -0,0 +1,95 @@ +const puppeteer = require('puppeteer'); +const axios = require('axios') +const path = require('path') + +const apiKey = "6hUfJdAartHTHhHc0WIRZYPWe" +// Spark Production ^ +// const apiKey = "p5fidpuedHaOlPnd8EcpxzQMG" +// Luminate Production ^ +const uid = "/\?uid\=7beg87y4-fh24-4929-3rt5-24kdn87s5241"; +const groupIds = [ + '855e2704-fd95-456f-a4eb-ce31c13b3072', + 'ba23909e-49fd-4458-81bc-99e9fe35faee', + 'efdfb1a5-58c8-4fd4-a31c-60946e73b6ed', +] + +const getAllGroups = async (num) => { + let i = 0 + loopIds(i) +} + +function loopIds(i) { + console.log("Loop Ids Function") + var num = 1 + id = groupIds[i] + courseOverview(id, i, num) +} + +async function courseOverview(id, i, num) { + // console.log("Course Overview Function") + const activity = new Array(); + const url = "https://walmart.northpass.com/app/courses/"; + const browser = await puppeteer.launch(); + const page = await browser.newPage(); + const course = url+id+uid; + console.log(course) + if (course.includes("undefined")) { + console.log("Error - Undefined UUID. Possibly end of list. Exiting.") + await browser.close(); + } else { + await page.setViewport({ width:390, height:844 }) + // await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3419.0 Safari/537.36'); + await page.goto(course, { + waitUntil: 'load', + timeout: 0 }); + + await page.title(); + const [getXpath] = await page.$x('/html/body/div[1]/div'); + const resourcetitle = await page.evaluate(name => name.innerText, getXpath); + // console.log(resourcetitle) + const resource = resourcetitle.trim(); + console.log(resource); + + await page.screenshot({path: `${resource}_1.png`, fullPage:true }); + + const hrefs = await page.$$eval('a', as => as.map(a => a.href)); + var links = Object.entries(hrefs); + // console.log(links) + links.forEach(([key, value]) => { + var link = value; + if (link.includes("activities") && (link.includes(id))) { + // console.log("Activities Link") + // console.log(link); + activity.push(link); + } + }); + setTimeout(() => { + + }, 10000); + uniqueLinks = [...new Set(activity)]; + for (const link of uniqueLinks) { + + num = num+1 + const newlink = link+uid; + console.log("New Link: " + newlink) + await page.setViewport({ width: 390, height: 844 }) + await page.goto(newlink, { + waitUntil: 'load', + timeout: 0 }); + await page.screenshot({path: `${resource}_${num}.png`, fullPage: true }); + screenshots(newlink, resource, num, i); + } + await browser.close(); + i++; + loopIds(i); + } +} + +async function screenshots(newlink, resource, num, i) { + const browser = await puppeteer.launch(); + const page = await browser.newPage(); + await browser.close(); +} + +getAllGroups(); +