From f7492abb5655fd6da630f2652f5697c24cd3694c Mon Sep 17 00:00:00 2001 From: vsoch Date: Thu, 16 Jun 2022 17:05:52 -0600 Subject: [PATCH] testing new user agent and headers Signed-off-by: vsoch --- CHANGELOG.md | 1 + urlchecker/core/urlproc.py | 52 +++++++++++++------------------------- urlchecker/version.py | 2 +- 3 files changed, 19 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5957f9d..5ef8bb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and **Merged pull requests**. Critical items to know are: Referenced versions in headers are tagged on Github, in parentheses are for pypi. ## [vxx](https://github.com/urlstechie/urlschecker-python/tree/master) (master) + - updating user agent strings, adding additional headers (0.0.30) - adding type hints to code, more tests and logging bug fix (0.0.29) - decrease verbosity when filename is None (0.0.28) - don't exit and fail if no URLs to check (0.0.27) diff --git a/urlchecker/core/urlproc.py b/urlchecker/core/urlproc.py index 5785da0..80f8ef7 100644 --- a/urlchecker/core/urlproc.py +++ b/urlchecker/core/urlproc.py @@ -54,45 +54,26 @@ def get_user_agent() -> str: user agent string to include with User-Agent. """ agents = [ - ( - "Mozilla/5.0 (X11; Linux x86_64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/57.0.2987.110 " - "Safari/537.36" - ), # chrome - ( - "Mozilla/5.0 (X11; Linux x86_64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/61.0.3163.79 " - "Safari/537.36" - ), # chrome - ( - "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) " - "Gecko/20100101 " - "Firefox/55.0" - ), # firefox - ( - "Mozilla/5.0 (X11; Linux x86_64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/61.0.3163.91 " - "Safari/537.36" - ), # chrome - ( - "Mozilla/5.0 (X11; Linux x86_64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/62.0.3202.89 " - "Safari/537.36" - ), # chrome - ( - "Mozilla/5.0 (X11; Linux x86_64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/63.0.3239.108 " - "Safari/537.36" - ), # chrome + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; Trident/5.0)", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; MDDCJS)", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", + "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)", ] return random.choice(agents) +def get_browser_headers(): + return { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "Accept-Encoding": "gzip, deflate", + "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8", + "Dnt": "1", + } + + class UrlCheckResult: """ A UrlCheckResult is a basic class to hold a result for a filename. @@ -188,6 +169,7 @@ def check_urls( # Some sites will return 403 if it's not a "human" user agent user_agent = get_user_agent() headers = {"User-Agent": user_agent} + headers.update(get_browser_headers()) # check links for url in [url for url in urls if "http" in url]: diff --git a/urlchecker/version.py b/urlchecker/version.py index d33fbd7..5be34c3 100644 --- a/urlchecker/version.py +++ b/urlchecker/version.py @@ -7,7 +7,7 @@ """ -__version__ = "0.0.29" +__version__ = "0.0.30" AUTHOR = "Ayoub Malek, Vanessa Sochat" AUTHOR_EMAIL = "superkogito@gmail.com, vsochat@stanford.edu" NAME = "urlchecker"