Documentation Releases

Example

"Download many URLs in parallel."


import functools
import urllib.request
import futures



URLS = ['http://www.foxnews.com/',

        'http://www.cnn.com/',

        'http://europe.wsj.com/',

        'http://www.bbc.co.uk/',

        'http://some-made-up-domain.com/']


def load_url(url, timeout):

    return urllib.request.urlopen(url, timeout=timeout).read()


with futures.ThreadPoolExecutor(50) as executor:

   future_list = executor.run_to_futures(

           [functools.partial(load_url, url, 30) for url in URLS])


for url, future in zip(URLS, future_list):

    if future.exception() is not None:

        print '%r generated an exception: %s' % (url, future.exception())

    else:

        print '%r page is %d bytes' % (url, len(future.result()))