35 lines
978 B
Python
35 lines
978 B
Python
|
BOT_NAME = "apiscraper"
|
||
|
|
|
||
|
|
SPIDER_MODULES = ["apiscraper.spiders"]
|
||
|
|
NEWSPIDER_MODULE = "apiscraper.spiders"
|
||
|
|
|
||
|
|
# Obey robots.txt rules
|
||
|
|
ROBOTSTXT_OBEY = True
|
||
|
|
|
||
|
|
# plywright required settings
|
||
|
|
DOWNLOAD_HANDLERS = {
|
||
|
|
"http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
|
||
|
|
"https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
|
||
|
|
}
|
||
|
|
PLAYWRIGHT_LAUNCH_OPTIONS = {"headless":True}
|
||
|
|
PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT = 100000
|
||
|
|
|
||
|
|
# Set settings whose default value is deprecated to a future-proof value
|
||
|
|
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
|
||
|
|
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
|
||
|
|
FEED_EXPORT_ENCODING = "utf-8"
|
||
|
|
|
||
|
|
#ITEM_PIPELINES = {
|
||
|
|
# 'apiscraper.pipelines.ApiscraperPipeline':10000,
|
||
|
|
# }
|
||
|
|
|
||
|
|
#FEEDS = {
|
||
|
|
# 'northpass_api.json': {
|
||
|
|
#'pages' : {
|
||
|
|
# 'method' : 'string',
|
||
|
|
# 'endpoint' : 'string',
|
||
|
|
# 'headers' : 'headers',
|
||
|
|
# }
|
||
|
|
# }
|
||
|
|
# }
|