first commit

apiscraper/settings.py

@@ -0,0 +1,34 @@
+BOT_NAME = "apiscraper"
+
+SPIDER_MODULES = ["apiscraper.spiders"]
+NEWSPIDER_MODULE = "apiscraper.spiders"
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = True
+
+# plywright required settings
+DOWNLOAD_HANDLERS = {
+    "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
+    "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
+}
+PLAYWRIGHT_LAUNCH_OPTIONS = {"headless":True}
+PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT = 100000
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
+TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+FEED_EXPORT_ENCODING = "utf-8"
+
+#ITEM_PIPELINES = {
+#        'apiscraper.pipelines.ApiscraperPipeline':10000,
+#        }
+
+#FEEDS = {
+#        'northpass_api.json': {
+            #'pages' : {
+#            'method' : 'string',
+#            'endpoint' : 'string',
+#            'headers' : 'headers',
+#            }
+#           }
+#        }