Python Requests Stream Async

Question

I am trying to download large file from python requests library by setting the stream=True

But i want this function to be executed asynchronously and send response back to server with downloading in the background.

Here is my code

async def downloadFile(url, filename):
  r = requests.get(url, stream=True)
  with open(os.path.join('./files', filename), 'wb+') as f:
    for chunk in r.iter_content(chunk_size=1024):
        if chunk:
            f.write(chunk)
  # Creating same file name
  # with _done appended to know that file has been downloaded
  with open(os.path.join('./files', filename + '_done'), 'w+') as f: 
    f.close()
  await asyncio.sleep(1)

Calling this function from other function like this

# check if file exist in server
        if(os.path.exists(os.path.join('./files', fileName))):

            #file exist!!!

            #check if done file exist
            if(os.path.exists(os.path.join('./files', fileName + '_done'))):

                #done file exist
                self.redirect(self.request.protocol + "://" +
                              self.request.host + '/files/' + fileName)
            else:
                #done file not exist. Wait for 5 min more

                self.write('Wait 5 min')
                self.finish()
        else:
            # file doesnt exist. Initiate download
            self.write('Wait 5 min')
            self.finish()
            d = asyncio.ensure_future(downloadFile(
                fileRes, fileName))
            # loop = asyncio.get_event_loop()
            # loop.run_until_complete(d)

The problem is that the file is created but its size remains 0 and the file appended "_done" is never created. What am I doing wrong here?

Answer 1

Your code works for me. Maybe is it the resource you're trying to get that does not work.

You might want to try enabling debug for requests as suggested by @brennan, and/or add printouts to your code to follow what's happening:

>>> import requests
>>> import asyncio
>>> 
>>> 
>>> async def downloadFile(url, filename):
...   print(f"• downloadFile({url}, {filename})")
...   r = requests.get(url, stream=True)
...   print(f" → r: {r}")
...   with open(os.path.join('./files', filename), 'wb+') as f:
...     print(f" → f is opened: {f}")
...     for chunk in r.iter_content(chunk_size=1024):
...         print(f"  → chunk is: {chunk}")
...         if chunk:
...             f.write(chunk)
...   # Creating same file name
...   # with _done appended to know that file has been downloaded
...   with open(os.path.join('./files', filename + '_done'), 'w+') as f:
...     print(f" → creating output with _done")
...     f.close()
...   print(f" → wait 1")
...   await asyncio.sleep(1)
... 
>>> 
>>> 
>>> d = asyncio.ensure_future(downloadFile('https://xxx/yyy.jpg', 'test.jpg'))
>>> loop = asyncio.get_event_loop()
>>> loop.run_until_complete(d)
• downloadFile(https://xxx/yyy.jpg, test.jpg)
 → r: <Response [200]>
 → f is opened: <_io.BufferedRandom name='./files/test.jpg'>
  → chunk is: b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00\xff\xdb\x00C\x00\r\t\n\x0b\n\x08\r\x0b\n\x0b\x0e\x0e\r\x0f\x13....'
  → chunk is: ...
  ...
 → creating output with _done
 → wait 1

that would make your _done part of your code useless (you only need the printouts). And even the wait at the end (when it's done… it's done!).

async def downloadFile(url, filename):
  r = requests.get(url, stream=True)
  with open(os.path.join('./files', filename), 'wb+') as f:
    for chunk in r.iter_content(chunk_size=1024):
        if chunk:
            f.write(chunk)

Though maybe you might want to catch any possible issues happening connecting to the server and act accordingly:

async def downloadFile(url, filename):
  try:
    r = requests.get(url, stream=True)
    r.raise_for_status() # to raise on invalid statuses
    with open(os.path.join('./files', filename), 'wb+') as f:
      for chunk in r.iter_content(chunk_size=1024):
          if chunk:
              f.write(chunk)
  except requests.RequestException as err:
    # do something smart when that exception occurs!
    print(f"Exception has occured: {err}")