From 85815f63a7f6f1adab334e80c1550ed591e289b0 Mon Sep 17 00:00:00 2001 From: Cooper Date: Sat, 13 Oct 2018 16:45:49 -0500 Subject: [PATCH] Add README.md and response_delay.py Added a READM.ME and a new script `response_delay.py`, which contains three functions: 1. random_user_agent: Takes in a list of user agents and returns a random one. 2. response_delay: Sets a response delay between requests, based on a pre-defined randomized wait time and how long the site takes to reply. 3. fetch_urls: Fetches the list of urls, applying the proper headers (user agents) and delays between requests (response_delay). Print how long each request takes. This is one of my first 'real' pull requests during Haktoberfest -- hopefully I am doing things correctly :smile:. --- response_delay/README.md | 11 +++++++ response_delay/response_delay.py | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 response_delay/README.md create mode 100644 response_delay/response_delay.py diff --git a/response_delay/README.md b/response_delay/README.md new file mode 100644 index 0000000..8f8cebd --- /dev/null +++ b/response_delay/README.md @@ -0,0 +1,11 @@ +# Fetching multiple urls and dynamically waiting between requests (response_delay) + +### About: +A python function that fetches urls based off a fed in list, and allows users to dynamically wait between successive HTTP requests (I use this when I'm web-scraping and need to be "kind" to the website). + +### Getting started: +- Download and install python3.6 from [here](https://www.python.org/downloads/) +- pip install requests + +### Getting help: +If your are facing any problem please raise an issue. diff --git a/response_delay/response_delay.py b/response_delay/response_delay.py new file mode 100644 index 0000000..c4a6e5c --- /dev/null +++ b/response_delay/response_delay.py @@ -0,0 +1,56 @@ +#!/usr/bin/python +""" +Python function to dynamically wait between requests calls +""" +#imports +import time +from random import randint, sample, choice +import requests + +USER_AGENTS = [ + {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 Safari/600.8.9'}, + {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36'}, + {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36'}, + {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'}, + {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'}, + {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1'} +] + +def random_user_agent(user_agents): + """(dict) -> str + + Takes in a dict of user_agents and returns one at random.""" + new_user_agent = choice(user_agents) + return new_user_agent + +def response_delay(t1): + """(int) -> int + + Gets the response delay, based on the randomized wait times you provide and the websites response time. + As of right now, we have the wait times set to wait ~7-8 seconds on average, and if the website responds in under 7 seconds, + then wait ~5 seconds more. These numbers can be easily changed.""" + response_time = time.time() - t1 + wait_time = randint(5, 10) - response_time + time.sleep(wait_time if wait_time > 7 else randint(3, 7)) + + +def fetch_urls(urls): + """(list) -> dict + + Fetches the fed in list of urls and applies the proper user_agents and dynamic wait time to the series of requests.""" + t0 = time.time() + total_requests = 0 + for url in urls: + try: + t1 = time.time() + response = requests.get(url, headers=random_user_agent(USER_AGENTS), timeout=60) + response_delay(t1) + total_requests += 1 + elapsed_time = time.time() - t0 + print(f'{total_requests}/{len(urls)}, {float(elapsed_time):.02f} requests/s') + response = response.text # Can do anything with this response. I usually turn it into a BeautifulSoup object before parsing the HTML. + except Exception as e: + print(e) + +urls = ['https://google.com', 'https://cnn.com', 'https://netflix.com'] +fetch_urls(urls)