phantop
/
quatalog-scraper
mirror of https://github.com/quatalog/quatalog.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix crashing without timeout arg and re-add --headless

This commit is contained in:
powe97 2024-03-01 00:29:34 -05:00
parent b0acd0e745
commit 1a4542e20e
No known key found for this signature in database
GPG Key ID: 7D1663B10978D1BA
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
transfer_scraper/main.py
View File

@ -115,18 +115,19 @@ def main():
transfer_json_path = sys.argv[1]
state_json_path = sys.argv[2]
timeout_seconds = int(sys.argv[3] or 120) * 60
timeout_seconds = int(sys.argv[3] if len(sys.argv) == 4 else 120) * 60
# Set up timeout so that the GH action does not run forever, pretend it's ^C
print(f"Setting timeout to {timeout_seconds} seconds", file=sys.stderr)
signal(SIGALRM, lambda a, b: raise_(KeyboardInterrupt))
alarm(timeout_seconds)
options = webdriver.FirefoxOptions()
options.add_argument("--headless")
user_agent = UserAgent().random
print(f"Using randomized user agent {user_agent}", file=sys.stderr)
if sys.argv[-1] != "gui":
options.add_argument("--headless")
options.set_preference("general.useragent.override", user_agent)
print(f"Using randomized user agent {user_agent}", file=sys.stderr)
driver = webdriver.Firefox(options=options)
driver.get(