From c17daddc44380a3304fcd12552409bfc46123bd4 Mon Sep 17 00:00:00 2001 From: Norm Rasmussen Date: Fri, 21 Jun 2024 15:52:31 -0400 Subject: [PATCH] Script for Anthology and getting courses & props --- .../get_courses_w_descriptions_or_props.py | 82 +++++++++++++ Scripts/Walmart/single_resource.js | 2 +- reactive-compose.yml | 111 ++++++++++++++++++ 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 Scripts/API_Tests/get_courses_w_descriptions_or_props.py create mode 100644 reactive-compose.yml diff --git a/Scripts/API_Tests/get_courses_w_descriptions_or_props.py b/Scripts/API_Tests/get_courses_w_descriptions_or_props.py new file mode 100644 index 00000000..2d6b2967 --- /dev/null +++ b/Scripts/API_Tests/get_courses_w_descriptions_or_props.py @@ -0,0 +1,82 @@ +""" +Small script to grab the list of courses from Northpass API +along with their status & description and write them to a CSV. +""" + +import requests +import pandas as pd +import Apikeys + +APIKEY = Apikeys.ANTHOLOGY +COURSES = [] +HEADERS = {"accept": "application/json", "X-Api-Key": APIKEY} + + +def get_course(): + """ + This function paginates through API responses to get the courses information + """ + + count = 0 + courses = [] + course_dict = {} + + while True: + count += 1 + url = f"https://api.northpass.com/v2/courses?page={count}" + response = requests.get(url, headers=HEADERS) + data = response.json() + nextlink = data["links"] + + for response in data["data"]: + status = response["attributes"]["status"] + # if status == "live": + # if name in COURSES: + uuid = response["id"] + name = response["attributes"]["name"] + # full_description = response["attributes"]["full_description"] + cprop = get_props(uuid) + course_dict = { + "id": uuid, + "name": name, + "status": status, + "product_names": cprop, + # "full_description": full_description, + } + print(course_dict) + + try: + courses.append(course_dict) + except TypeError as e: + print(f"Error: {e}") + finally: + write_to_csv(courses) + pass + else: + pass + + if "next" not in nextlink: + break + + +def get_props(uuid): + cprop_url = f"https://api.northpass.com/v2/properties/courses/{uuid}?filter[]" + cprop_resp = requests.get(cprop_url, headers=HEADERS) + cdata = cprop_resp.json() + + for ckey, cval in cdata["data"]["attributes"]["properties"].items(): + if ckey == "product_names_for_course_cards": + return cval + + +def write_to_csv(courses): + """ + Function to write the list of dictionaries to a CSV using pandas. + Takes on parameter, the list of courses. + """ + df = pd.DataFrame.from_dict(courses) + df.to_csv("/Users/normrasmussen/Downloads/courses_with_property.csv") + + +if __name__ == "__main__": + get_course() diff --git a/Scripts/Walmart/single_resource.js b/Scripts/Walmart/single_resource.js index ded3e6a0..ff3d4ae6 100644 --- a/Scripts/Walmart/single_resource.js +++ b/Scripts/Walmart/single_resource.js @@ -8,7 +8,7 @@ const apiKey = "6hUfJdAartHTHhHc0WIRZYPWe" // Luminate Production ^ const uid = "/\?uid\=7beg87y4-fh24-4929-3rt5-24kdn87s5241"; const groupIds = [ - 'e6afa0b1-a7f8-4e34-a4e7-c1c4c45c60a0' + '64d18ece-b076-4e26-9e13-e15c321d52be' ] const getAllGroups = async (num) => { diff --git a/reactive-compose.yml b/reactive-compose.yml new file mode 100644 index 00000000..6289584e --- /dev/null +++ b/reactive-compose.yml @@ -0,0 +1,111 @@ +version: "3.8" + +# In this Docker Compose example, it assumes that you maintain a reverse proxy externally (or chose not to). +# The only two exposed ports here are from minio (:9000) and the app itself (:3000). +# If these ports are changed, ensure that the env vars passed to the app are also changed accordingly. + +services: + # Database (Postgres) + postgres: + image: postgres:16-alpine + restart: unless-stopped + volumes: + - postgres_data:/var/lib/postgresql/data + environment: + POSTGRES_DB: pg-reactive + POSTGRES_USER: pg-reactive + POSTGRES_PASSWORD: reactiveresume + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 10s + timeout: 5s + retries: 5 + + # Storage (for image uploads) + minio: + image: minio/minio + restart: unless-stopped + command: server /data + ports: + - "9000:9000" + volumes: + - minio_data:/data + environment: + MINIO_ROOT_USER: rr-minioadmin + MINIO_ROOT_PASSWORD: minioreactive + + # Chrome Browser (for printing and previews) + chrome: + image: ghcr.io/browserless/chromium:latest + restart: unless-stopped + environment: + TIMEOUT: 10000 + CONCURRENT: 10 + TOKEN: chrome_token + EXIT_ON_HEALTH_FAILURE: true + PRE_REQUEST_HEALTH_CHECK: true + + app: + image: amruthpillai/reactive-resume:latest + restart: unless-stopped + ports: + - "3000:3000" + depends_on: + - postgres + - minio + - chrome + environment: + # -- Environment Variables -- + PORT: 3000 + NODE_ENV: production + + # -- URLs -- + PUBLIC_URL: http://192.168.200.73:3000 + STORAGE_URL: http://192.168.200.73:9000/default + + # -- Printer (Chrome) -- + CHROME_TOKEN: chrome_token + CHROME_URL: ws://chrome:3000 + + # -- Database (Postgres) -- + DATABASE_URL: postgresql://pg-reactive:reactiveresume@pg-postgres:5432/postgres + + # -- Auth -- + ACCESS_TOKEN_SECRET: c13fdfdacfdeb2b5185dffe89ba4d152a2e71d5d36c691fbc0a594e9695d992c87 + REFRESH_TOKEN_SECRET: d8665ce1f01b1dd2db85a53517c362a7668f025058e5218ee4e0d6fe233a81 + + # -- Emails -- + MAIL_FROM: no-reply-web@rsmsn.co + # SMTP_URL: smtp://user:pass@smtp:587 # Optional + + # -- Storage (Minio) -- + STORAGE_ENDPOINT: minio + STORAGE_PORT: 9000 + STORAGE_REGION: us-east-1 # Optional + STORAGE_BUCKET: default + STORAGE_ACCESS_KEY: minioadmin + STORAGE_SECRET_KEY: minioadmin + STORAGE_USE_SSL: false + STORAGE_SKIP_BUCKET_CHECK: false + + # -- Crowdin (Optional) -- + # CROWDIN_PROJECT_ID: + # CROWDIN_PERSONAL_TOKEN: + + # -- Email (Optional) -- + # DISABLE_SIGNUPS: false + # DISABLE_EMAIL_AUTH: false + + # -- GitHub (Optional) -- + # GITHUB_CLIENT_ID: github_client_id + # GITHUB_CLIENT_SECRET: github_client_secret + # GITHUB_CALLBACK_URL: http://192.168.200.73:3000/api/auth/github/callback + + # -- Google (Optional) -- + # GOOGLE_CLIENT_ID: google_client_id + # GOOGLE_CLIENT_SECRET: google_client_secret + # GOOGLE_CALLBACK_URL: http://192.168.200.73:3000/api/auth/google/callback + +volumes: + minio_data: + postgres_data: