phantop/quatalog-scraper
1
0
Fork 0
mirror of https://github.com/quatalog/quatalog.git synced 2024-11-08 08:04:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix crashing without timeout arg and re-add --headless

Browse source
This commit is contained in:
powe97 2024-03-01 00:29:34 -05:00
parent b0acd0e745
commit 1a4542e20e
No known key found for this signature in database
GPG key ID: 7D1663B10978D1BA
1 changed files with 5 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
transfer_scraper/main.py
View file

@ -115,18 +115,19 @@ def main():
transfer_json_path = sys.argv[1]
state_json_path = sys.argv[2]
timeout_seconds = int(sys.argv[3] or 120) * 60
timeout_seconds = int(sys.argv[3] if len(sys.argv) == 4 else 120) * 60
# Set up timeout so that the GH action does not run forever, pretend it's ^C
print(f"Setting timeout to {timeout_seconds} seconds", file=sys.stderr)
signal(SIGALRM, lambda a, b: raise_(KeyboardInterrupt))
alarm(timeout_seconds)
options = webdriver.FirefoxOptions()
options.add_argument("--headless")
user_agent = UserAgent().random
print(f"Using randomized user agent {user_agent}", file=sys.stderr)
if sys.argv[-1] != "gui":
options.add_argument("--headless")
options.set_preference("general.useragent.override", user_agent)
print(f"Using randomized user agent {user_agent}", file=sys.stderr)
driver = webdriver.Firefox(options=options)
driver.get(