first commit
This commit is contained in:
Normanras
BIN
apiscraper/spiders/.DS_Store
vendored
Normal file
BIN
apiscraper/spiders/.DS_Store
vendored
Normal file
Binary file not shown.
1
apiscraper/spiders/.null-ls_505405_dev_spider.py
Normal file
1
apiscraper/spiders/.null-ls_505405_dev_spider.py
Normal file
@ -0,0 +1 @@
|
||||
|
||||
4
apiscraper/spiders/__init__.py
Normal file
4
apiscraper/spiders/__init__.py
Normal file
@ -0,0 +1,4 @@
|
||||
# This package will contain the spiders of your Scrapy project
|
||||
#
|
||||
# Please refer to the documentation for information on how to create and manage
|
||||
# your spiders.
|
||||
BIN
apiscraper/spiders/__pycache__/__init__.cpython-311.pyc
Normal file
BIN
apiscraper/spiders/__pycache__/__init__.cpython-311.pyc
Normal file
Binary file not shown.
BIN
apiscraper/spiders/__pycache__/dev_spider.cpython-311.pyc
Normal file
BIN
apiscraper/spiders/__pycache__/dev_spider.cpython-311.pyc
Normal file
Binary file not shown.
BIN
apiscraper/spiders/api.db
Normal file
BIN
apiscraper/spiders/api.db
Normal file
Binary file not shown.
71
apiscraper/spiders/dev_spider.py
Normal file
71
apiscraper/spiders/dev_spider.py
Normal file
@ -0,0 +1,71 @@
|
||||
from pathlib import Path
|
||||
import scrapy
|
||||
from scrapy_playwright.page import PageMethod
|
||||
from apiscraper.items import EndpointItems
|
||||
|
||||
class ApiSpider(scrapy.Spider):
|
||||
name = "api"
|
||||
|
||||
def start_requests(self):
|
||||
urls = [
|
||||
"https://developers.northpass.com/reference/get_v2-activities",
|
||||
]
|
||||
for url in urls:
|
||||
yield scrapy.Request(url=url, meta=dict(
|
||||
playwright = True,
|
||||
playwright_include_page = True,
|
||||
playwright_page_methods =[PageMethod('wait_for_selector', '//*[@id="content"]/div[4]/nav'),
|
||||
],
|
||||
errback=self.errback,
|
||||
))
|
||||
|
||||
async def parse(self, response):
|
||||
page = response.meta["playwright_page"]
|
||||
await page.close()
|
||||
|
||||
northpass_endpoints = {}
|
||||
|
||||
end_item = EndpointItems()
|
||||
end_item['inclusions'] = []
|
||||
end_item['params'] = []
|
||||
for item in response.xpath('//*[@id="Explorer"]'):
|
||||
end_item['title'] = response.xpath('//*[@id="content"]/header[1]/div[1]/h1/text()').get()
|
||||
end_item['method'] = response.xpath('//*[@id="content"]/header[1]/div[2]/span[1]/text()').get()
|
||||
end_item['endpoint'] = response.xpath('//*[@id="content"]/header[1]/div[2]/span[2]/text()[2]').get()
|
||||
end_item['inclusions'].append(response.xpath('//*[starts-with(@id,"query")]/div/p/code/span[position() >= 1 and not(position() > 15)]/text()').get())
|
||||
#end_item['params'].append(response.xpath('//*[starts-with(@id,"path"]/div/div[1]/div/label/text()').get())
|
||||
if end_item['inclusions'] is not None:
|
||||
inclusions = end_item['inclusions']
|
||||
else:
|
||||
pass
|
||||
|
||||
northpass_endpoints = {
|
||||
end_item['title'] : {
|
||||
'method' : end_item['method'],
|
||||
'endpoint' : end_item['endpoint'],
|
||||
'inclusions' : inclusions
|
||||
}
|
||||
}
|
||||
|
||||
yield northpass_endpoints
|
||||
yield EndpointItems()
|
||||
|
||||
self.log(northpass_endpoints)
|
||||
|
||||
next_page = response.xpath('//*[@id="content"]/div[4]/nav/a[2]/@href').get()
|
||||
self.log(f"The next page is {next_page}")
|
||||
|
||||
if next_page is not None:
|
||||
nexturl = response.urljoin(next_page)
|
||||
yield scrapy.Request(nexturl, meta=dict(
|
||||
playwright = True,
|
||||
playwright_include_page = True,
|
||||
playwright_page_methods =[
|
||||
PageMethod('wait_for_selector', '//*[@id="content"]/div[4]/nav'),
|
||||
],
|
||||
errback=self.errback,
|
||||
))
|
||||
|
||||
async def errback(self, failure):
|
||||
page = failure.request.meta["playwright_page"]
|
||||
await page.close()
|
||||
202
apiscraper/spiders/outputtest.json
Normal file
202
apiscraper/spiders/outputtest.json
Normal file
@ -0,0 +1,202 @@
|
||||
[
|
||||
{"List activities": {"method": "get", "endpoint": "/v2/activities"}},
|
||||
{},
|
||||
{"Fetch an activity": {"method": "get", "endpoint": "/v2/activities/"}},
|
||||
{},
|
||||
{"BambooHR configuration": {"method": "get", "endpoint": "/v2/apps/bamboo_hr"}},
|
||||
{},
|
||||
{"Update BambooHR configuration": {"method": "put", "endpoint": "/v2/apps/bamboo_hr"}},
|
||||
{},
|
||||
{"Delete BambooHR configuration": {"method": "delete", "endpoint": "/v2/apps/bamboo_hr"}},
|
||||
{},
|
||||
{"List assignment submissions": {"method": "get", "endpoint": "/v2/assignments/"}},
|
||||
{},
|
||||
{"Fetch an assignment submission": {"method": "get", "endpoint": "/v2/assignments/"}},
|
||||
{},
|
||||
{"List all assignments": {"method": "get", "endpoint": "/v2/assignments"}},
|
||||
{},
|
||||
{"Bulk add courses to groups": {"method": "post", "endpoint": "/v2/bulk/groups/courses"}},
|
||||
{},
|
||||
{"Bulk create groups for a school": {"method": "post", "endpoint": "/v2/bulk/groups"}},
|
||||
{},
|
||||
{"Bulk add people to groups": {"method": "post", "endpoint": "/v2/bulk/people/membership"}},
|
||||
{},
|
||||
{"Bulk enroll people to courses [deprecated]": {"method": "post", "endpoint": "/v2/bulk/enrollments"}},
|
||||
{},
|
||||
{"Bulk enroll people to courses": {"method": "post", "endpoint": "/v2/bulk/people/course"}},
|
||||
{},
|
||||
{"Bulk invite people to a school": {"method": "post", "endpoint": "/v2/bulk/people"}},
|
||||
{},
|
||||
{"Bulk resend school inviation to people": {"method": "post", "endpoint": "/v2/bulk/people/resend_invitation"}},
|
||||
{},
|
||||
{"List categories": {"method": "get", "endpoint": "/v2/categories"}},
|
||||
{},
|
||||
{"Create a category": {"method": "post", "endpoint": "/v2/categories"}},
|
||||
{},
|
||||
{"Fetch a category": {"method": "get", "endpoint": "/v2/categories/"}},
|
||||
{},
|
||||
{"Update a category": {"method": "put", "endpoint": "/v2/categories/"}},
|
||||
{},
|
||||
{"Delete a category": {"method": "delete", "endpoint": "/v2/categories/"}},
|
||||
{},
|
||||
{"Resend communication": {"method": "post", "endpoint": "/v2/communications/deliveries/"}},
|
||||
{},
|
||||
{"List deliveries": {"method": "get", "endpoint": "/v2/communications/deliveries/"}},
|
||||
{},
|
||||
{"Update attendance confirmation notification": {"method": "put", "endpoint": "/v2/communications/emails/attendance_confirmation"}},
|
||||
{},
|
||||
{"Update courses incomplete notification": {"method": "put", "endpoint": "/v2/communications/emails/courses_incomplete_notification"}},
|
||||
{},
|
||||
{"Update new courses notification": {"method": "put", "endpoint": "/v2/communications/emails/new_courses_notification"}},
|
||||
{},
|
||||
{"Update training session registration notification": {"method": "put", "endpoint": "/v2/communications/emails/training_session_registration_confirmation"}},
|
||||
{},
|
||||
{"List activities for a course": {"method": "get", "endpoint": "/v2/courses/"}},
|
||||
{},
|
||||
{"List groups not yet associated with course": {"method": "get", "endpoint": "/v2/courses/"}},
|
||||
{},
|
||||
{"List people not yet enrolled with course": {"method": "get", "endpoint": "/v2/courses/"}},
|
||||
{},
|
||||
{"List enrollments for a course": {"method": "get", "endpoint": "/v2/courses/"}},
|
||||
{},
|
||||
{"Retake a course": {"method": "post", "endpoint": "/v2/courses/"}},
|
||||
{},
|
||||
{"List courses": {"method": "get", "endpoint": "/v2/courses"}},
|
||||
{},
|
||||
{"List credential achievements": {"method": "get", "endpoint": "/v2/credentials/"}},
|
||||
{},
|
||||
{"List credential courses": {"method": "get", "endpoint": "/v2/credentials/"}},
|
||||
{},
|
||||
{"Delete course credential": {"method": "delete", "endpoint": "/v2/credentials/"}},
|
||||
{},
|
||||
{"List credentials": {"method": "get", "endpoint": "/v2/credentials"}},
|
||||
{},
|
||||
{"Delete a credential": {"method": "delete", "endpoint": "/v2/credentials/"}},
|
||||
{},
|
||||
{"List custom templates": {"method": "get", "endpoint": "/v2/custom_templates"}},
|
||||
{},
|
||||
{"Create a custom template": {"method": "post", "endpoint": "/v2/custom_templates"}},
|
||||
{},
|
||||
{"Delete a custom template": {"method": "delete", "endpoint": "/v2/custom_templates/"}},
|
||||
{},
|
||||
{"List email senders": {"method": "get", "endpoint": "/v2/email_domains/"}},
|
||||
{},
|
||||
{"Update an email sender": {"method": "put", "endpoint": "/v2/email_domains/"}},
|
||||
{},
|
||||
{"Delete an email sender": {"method": "delete", "endpoint": "/v2/email_domains/"}},
|
||||
{},
|
||||
{"List email domains": {"method": "get", "endpoint": "/v2/email_domains"}},
|
||||
{},
|
||||
{"Create an email domain": {"method": "post", "endpoint": "/v2/email_domains"}},
|
||||
{},
|
||||
{"Delete an email domain": {"method": "delete", "endpoint": "/v2/email_domains/"}},
|
||||
{},
|
||||
{"List events": {"method": "get", "endpoint": "/v2/events"}},
|
||||
{},
|
||||
{"List group's courses": {"method": "get", "endpoint": "/v2/groups/"}},
|
||||
{},
|
||||
{"List group's memberships": {"method": "get", "endpoint": "/v2/groups/"}},
|
||||
{},
|
||||
{"Add courses to a group": {"method": "post", "endpoint": "/v2/groups/"}},
|
||||
{},
|
||||
{"Add people to a group": {"method": "post", "endpoint": "/v2/groups/"}},
|
||||
{},
|
||||
{"List groups": {"method": "get", "endpoint": "/v2/groups"}},
|
||||
{},
|
||||
{"Create a group": {"method": "post", "endpoint": "/v2/groups"}},
|
||||
{},
|
||||
{"Fetch a group": {"method": "get", "endpoint": "/v2/groups/"}},
|
||||
{},
|
||||
{"Update a group": {"method": "put", "endpoint": "/v2/groups/"}},
|
||||
{},
|
||||
{"Delete a group": {"method": "delete", "endpoint": "/v2/groups/"}},
|
||||
{},
|
||||
{"Deactivate a person": {"method": "post", "endpoint": "/v2/people/"}},
|
||||
{},
|
||||
{"Reactivate a person": {"method": "delete", "endpoint": "/v2/people/"}},
|
||||
{},
|
||||
{"Associate a person with a course": {"method": "post", "endpoint": "/v2/people/"}},
|
||||
{},
|
||||
{"Remove a person from a course": {"method": "delete", "endpoint": "/v2/people/"}},
|
||||
{},
|
||||
{"Add groups to a person": {"method": "post", "endpoint": "/v2/people/"}},
|
||||
{},
|
||||
{"Remove a person from a group": {"method": "delete", "endpoint": "/v2/people/"}},
|
||||
{},
|
||||
{"List people": {"method": "get", "endpoint": "/v2/people"}},
|
||||
{},
|
||||
{"Create a person": {"method": "post", "endpoint": "/v2/people"}},
|
||||
{},
|
||||
{"Fetch a person": {"method": "get", "endpoint": "/v2/people/"}},
|
||||
{},
|
||||
{"Update a person": {"method": "put", "endpoint": "/v2/people/"}},
|
||||
{},
|
||||
{"Delete a person": {"method": "delete", "endpoint": "/v2/people/"}},
|
||||
{},
|
||||
{"List question banks": {"method": "get", "endpoint": "/v2/question_banks"}},
|
||||
{},
|
||||
{"Create a question bank": {"method": "post", "endpoint": "/v2/question_banks"}},
|
||||
{},
|
||||
{"Fetch a question bank": {"method": "get", "endpoint": "/v2/question_banks/"}},
|
||||
{},
|
||||
{"List learner quiz answers": {"method": "get", "endpoint": "/v2/quiz_attempts/"}},
|
||||
{},
|
||||
{"List quizzes": {"method": "get", "endpoint": "/v2/quizzes"}},
|
||||
{},
|
||||
{"List all assignment submissions": {"method": "get", "endpoint": "/v2/submissions"}},
|
||||
{},
|
||||
{"List all configured webhook endpoints": {"method": "get", "endpoint": "/v2/webhook_endpoints"}},
|
||||
{},
|
||||
{"Creates new webhook endpoint": {"method": "post", "endpoint": "/v2/webhook_endpoints"}},
|
||||
{},
|
||||
{"Updates webhook endpoint": {"method": "put", "endpoint": "/v2/webhook_endpoints/"}},
|
||||
{},
|
||||
{"Delete a webhook endpoint": {"method": "delete", "endpoint": "/v2/webhook_endpoints/"}},
|
||||
{},
|
||||
{"List all sent webhooks": {"method": "get", "endpoint": "/v2/webhooks"}},
|
||||
{},
|
||||
{"Bulk fill courses property values": {"method": "post", "endpoint": "/v2/properties/courses/bulk"}},
|
||||
{},
|
||||
{"Bulk clear courses property values": {"method": "delete", "endpoint": "/v2/properties/courses/bulk"}},
|
||||
{},
|
||||
{"Bulk fill people property values": {"method": "post", "endpoint": "/v2/properties/people/bulk"}},
|
||||
{},
|
||||
{"Bulk clear people property values": {"method": "delete", "endpoint": "/v2/properties/people/bulk"}},
|
||||
{},
|
||||
{"List properties available for courses": {"method": "get", "endpoint": "/v2/properties/courses/properties"}},
|
||||
{},
|
||||
{"Get available course properties with values": {"method": "get", "endpoint": "/v2/properties/courses/"}},
|
||||
{},
|
||||
{"List created property definitions": {"method": "get", "endpoint": "/v2/properties/conditional"}},
|
||||
{},
|
||||
{"Create conditional property with mapping": {"method": "post", "endpoint": "/v2/properties/conditional"}},
|
||||
{},
|
||||
{"Delete conditional property": {"method": "delete", "endpoint": "/v2/properties/conditional/"}},
|
||||
{},
|
||||
{"Update conditional property": {"method": "patch", "endpoint": "/v2/properties/conditional/"}},
|
||||
{},
|
||||
{"List properties available for people": {"method": "get", "endpoint": "/v2/properties/people/properties"}},
|
||||
{},
|
||||
{"Get available person properties with values": {"method": "get", "endpoint": "/v2/properties/people/"}},
|
||||
{},
|
||||
{"List unique values for property": {"method": "get", "endpoint": "/v2/properties/property_definitions/"}},
|
||||
{},
|
||||
{"List created property definitions": {"method": "get", "endpoint": "/v2/properties/property_definitions"}},
|
||||
{},
|
||||
{"Create property definition": {"method": "post", "endpoint": "/v2/properties/property_definitions"}},
|
||||
{},
|
||||
{"Get property definition": {"method": "get", "endpoint": "/v2/properties/property_definitions/"}},
|
||||
{},
|
||||
{"Delete property definition": {"method": "delete", "endpoint": "/v2/properties/property_definitions/"}},
|
||||
{},
|
||||
{"Create property definition": {"method": "patch", "endpoint": "/v2/properties/property_definitions/"}},
|
||||
{},
|
||||
{"List properties available for school": {"method": "get", "endpoint": "/v2/properties/school/properties"}},
|
||||
{},
|
||||
{"Get available school properties with values": {"method": "get", "endpoint": "/v2/properties/school"}},
|
||||
{},
|
||||
{"Clear school properties with values": {"method": "delete", "endpoint": "/v2/properties/school"}},
|
||||
{},
|
||||
{"Fill school properties with values": {"method": "patch", "endpoint": "/v2/properties/school"}},
|
||||
{}
|
||||
]
|
||||
Reference in New Issue
Block a user