import requests import pandas as pd import Apikeys apiKey = Apikeys.walmartprod course_dict = {} def get_course(): count = 0 courses = [] while True: count += 1 url = f"https://api2.northpass.com/v2/courses?page={count}" headers = {"accept": "application/json", "X-Api-Key": apiKey} response = requests.get(url, headers=headers) data = response.json() nextlink = data["links"] for response in data["data"]: status = response["attributes"]["status"] uuid = response["id"] name = response["attributes"]["name"] course_dict = { "id": uuid, "name": name, "status": status, "url": f"https://walmart.northpass.com/app/courses/{uuid}", } # FIX: Up until here, each course gets read to the terminal. # FIX: After this, something is being overwritten. cat_id = response["relationships"]["categories"]["data"] get_cat_name(cat_id, course_dict, courses) if "next" not in nextlink: break def get_cat_name(cat_id, course_dict, courses): headers = {"accept": "application/json", "X-Api-Key": apiKey} cats = [] if len(cat_id) == 0: pass elif len(cat_id) == 1: categoryid = cat_id[0]["id"] url = f"https://api2.northpass.com/v2/categories/{categoryid}" cat_resp = requests.get(url, headers=headers) cat_data = cat_resp.json() cat_name = cat_data["data"]["attributes"]["name"] print(cat_name) cats.append(cat_name) course_dict.update({"categories": cats}) else: for item in cat_id: categoryid = item["id"] url = f"https://api2.northpass.com/v2/categories/{categoryid}" cat_resp = requests.get(url, headers=headers) cat_data = cat_resp.json() cat_name = cat_data["data"]["attributes"]["name"] cats.append(cat_name) course_dict.update({"categories": cats}) try: courses.append(course_dict) except TypeError as e: print(f"Error: {e}") finally: write_to_csv(courses) def write_to_csv(courses): df = pd.DataFrame.from_dict(courses) df.to_csv("/Users/normrasmussen/Downloads/walmart_courses.csv") if __name__ == "__main__": get_course()