Script for Anthology and getting courses & props

This commit is contained in:
Norm Rasmussen
2024-06-21 15:52:31 -04:00
parent 2e82486eb8
commit c17daddc44
3 changed files with 194 additions and 1 deletions

View File

@ -0,0 +1,82 @@
"""
Small script to grab the list of courses from Northpass API
along with their status & description and write them to a CSV.
"""
import requests
import pandas as pd
import Apikeys
APIKEY = Apikeys.ANTHOLOGY
COURSES = []
HEADERS = {"accept": "application/json", "X-Api-Key": APIKEY}
def get_course():
"""
This function paginates through API responses to get the courses information
"""
count = 0
courses = []
course_dict = {}
while True:
count += 1
url = f"https://api.northpass.com/v2/courses?page={count}"
response = requests.get(url, headers=HEADERS)
data = response.json()
nextlink = data["links"]
for response in data["data"]:
status = response["attributes"]["status"]
# if status == "live":
# if name in COURSES:
uuid = response["id"]
name = response["attributes"]["name"]
# full_description = response["attributes"]["full_description"]
cprop = get_props(uuid)
course_dict = {
"id": uuid,
"name": name,
"status": status,
"product_names": cprop,
# "full_description": full_description,
}
print(course_dict)
try:
courses.append(course_dict)
except TypeError as e:
print(f"Error: {e}")
finally:
write_to_csv(courses)
pass
else:
pass
if "next" not in nextlink:
break
def get_props(uuid):
cprop_url = f"https://api.northpass.com/v2/properties/courses/{uuid}?filter[]"
cprop_resp = requests.get(cprop_url, headers=HEADERS)
cdata = cprop_resp.json()
for ckey, cval in cdata["data"]["attributes"]["properties"].items():
if ckey == "product_names_for_course_cards":
return cval
def write_to_csv(courses):
"""
Function to write the list of dictionaries to a CSV using pandas.
Takes on parameter, the list of courses.
"""
df = pd.DataFrame.from_dict(courses)
df.to_csv("/Users/normrasmussen/Downloads/courses_with_property.csv")
if __name__ == "__main__":
get_course()

View File

@ -8,7 +8,7 @@ const apiKey = "6hUfJdAartHTHhHc0WIRZYPWe"
// Luminate Production ^
const uid = "/\?uid\=7beg87y4-fh24-4929-3rt5-24kdn87s5241";
const groupIds = [
'e6afa0b1-a7f8-4e34-a4e7-c1c4c45c60a0'
'64d18ece-b076-4e26-9e13-e15c321d52be'
]
const getAllGroups = async (num) => {

111
reactive-compose.yml Normal file
View File

@ -0,0 +1,111 @@
version: "3.8"
# In this Docker Compose example, it assumes that you maintain a reverse proxy externally (or chose not to).
# The only two exposed ports here are from minio (:9000) and the app itself (:3000).
# If these ports are changed, ensure that the env vars passed to the app are also changed accordingly.
services:
# Database (Postgres)
postgres:
image: postgres:16-alpine
restart: unless-stopped
volumes:
- postgres_data:/var/lib/postgresql/data
environment:
POSTGRES_DB: pg-reactive
POSTGRES_USER: pg-reactive
POSTGRES_PASSWORD: reactiveresume
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"]
interval: 10s
timeout: 5s
retries: 5
# Storage (for image uploads)
minio:
image: minio/minio
restart: unless-stopped
command: server /data
ports:
- "9000:9000"
volumes:
- minio_data:/data
environment:
MINIO_ROOT_USER: rr-minioadmin
MINIO_ROOT_PASSWORD: minioreactive
# Chrome Browser (for printing and previews)
chrome:
image: ghcr.io/browserless/chromium:latest
restart: unless-stopped
environment:
TIMEOUT: 10000
CONCURRENT: 10
TOKEN: chrome_token
EXIT_ON_HEALTH_FAILURE: true
PRE_REQUEST_HEALTH_CHECK: true
app:
image: amruthpillai/reactive-resume:latest
restart: unless-stopped
ports:
- "3000:3000"
depends_on:
- postgres
- minio
- chrome
environment:
# -- Environment Variables --
PORT: 3000
NODE_ENV: production
# -- URLs --
PUBLIC_URL: http://192.168.200.73:3000
STORAGE_URL: http://192.168.200.73:9000/default
# -- Printer (Chrome) --
CHROME_TOKEN: chrome_token
CHROME_URL: ws://chrome:3000
# -- Database (Postgres) --
DATABASE_URL: postgresql://pg-reactive:reactiveresume@pg-postgres:5432/postgres
# -- Auth --
ACCESS_TOKEN_SECRET: c13fdfdacfdeb2b5185dffe89ba4d152a2e71d5d36c691fbc0a594e9695d992c87
REFRESH_TOKEN_SECRET: d8665ce1f01b1dd2db85a53517c362a7668f025058e5218ee4e0d6fe233a81
# -- Emails --
MAIL_FROM: no-reply-web@rsmsn.co
# SMTP_URL: smtp://user:pass@smtp:587 # Optional
# -- Storage (Minio) --
STORAGE_ENDPOINT: minio
STORAGE_PORT: 9000
STORAGE_REGION: us-east-1 # Optional
STORAGE_BUCKET: default
STORAGE_ACCESS_KEY: minioadmin
STORAGE_SECRET_KEY: minioadmin
STORAGE_USE_SSL: false
STORAGE_SKIP_BUCKET_CHECK: false
# -- Crowdin (Optional) --
# CROWDIN_PROJECT_ID:
# CROWDIN_PERSONAL_TOKEN:
# -- Email (Optional) --
# DISABLE_SIGNUPS: false
# DISABLE_EMAIL_AUTH: false
# -- GitHub (Optional) --
# GITHUB_CLIENT_ID: github_client_id
# GITHUB_CLIENT_SECRET: github_client_secret
# GITHUB_CALLBACK_URL: http://192.168.200.73:3000/api/auth/github/callback
# -- Google (Optional) --
# GOOGLE_CLIENT_ID: google_client_id
# GOOGLE_CLIENT_SECRET: google_client_secret
# GOOGLE_CALLBACK_URL: http://192.168.200.73:3000/api/auth/google/callback
volumes:
minio_data:
postgres_data: