Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions requests_toolbelt/downloadutils/stream.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
"""Utilities for dealing with streamed requests."""
import errno
import os.path
import re

Expand All @@ -13,6 +14,9 @@
_QUOTED_STRING_RE)
)
_DEFAULT_CHUNKSIZE = 512
# Create the file only if it does not exist yet, as one atomic step.
_EXCLUSIVE_CREATE_FLAGS = (os.O_WRONLY | os.O_CREAT | os.O_EXCL |
getattr(os, 'O_BINARY', 0))


def _get_filename(content_disposition):
Expand Down Expand Up @@ -163,9 +167,17 @@ def stream_response_to_file(response, path=None, chunksize=_DEFAULT_CHUNKSIZE):
filename = getattr(fd, 'name', None)
else:
filename = get_download_file_path(response, path)
if os.path.exists(filename):
try:
descriptor = os.open(filename, _EXCLUSIVE_CREATE_FLAGS, 0o666)
except OSError as error:
if error.errno != errno.EEXIST:
raise
raise exc.StreamingError("File already exists: %s" % filename)
fd = open(filename, 'wb')
try:
fd = os.fdopen(descriptor, 'wb')
except BaseException:
os.close(descriptor)
raise

for chunk in response.iter_content(chunk_size=chunksize):
fd.write(chunk)
Expand Down
71 changes: 71 additions & 0 deletions tests/test_download_creation_race.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
"""A competing creator must never lose its existing bytes."""
import os
from io import BytesIO

try:
from unittest.mock import Mock
except ImportError:
from mock import Mock

import pytest

from requests_toolbelt.downloadutils import stream
from requests_toolbelt.exceptions import StreamingError


def response():
result = Mock()
result.iter_content.return_value = [b"download"]
return result


def test_competing_file_creation_preserves_winner(tmp_path, monkeypatch):
target = tmp_path / "download.bin"
real_os_open = os.open

def race(path, flags, mode=0o777):
with open(path, "wb") as winner:
winner.write(b"competing file")
return real_os_open(path, flags, mode)

monkeypatch.setattr(stream.os, "open", race)
with pytest.raises(StreamingError, match="File already exists"):
stream.stream_response_to_file(response(), str(target))
assert target.read_bytes() == b"competing file"


def test_descriptor_is_closed_when_wrapping_fails(tmp_path, monkeypatch):
target = tmp_path / "download.bin"
real_os_open = os.open
real_os_close = os.close
opened = []
closed = []

def record_open(path, flags, mode=0o777):
opened.append(real_os_open(path, flags, mode))
return opened[-1]

def record_close(descriptor):
closed.append(descriptor)
real_os_close(descriptor)

def fail_fdopen(descriptor, mode):
raise MemoryError("synthetic fdopen failure")

monkeypatch.setattr(stream.os, "open", record_open)
monkeypatch.setattr(stream.os, "close", record_close)
monkeypatch.setattr(stream.os, "fdopen", fail_fdopen)
with pytest.raises(MemoryError, match="synthetic fdopen failure"):
stream.stream_response_to_file(response(), str(target))
assert len(opened) == 1
assert closed == opened


def test_new_file_and_caller_owned_stream(tmp_path):
target = tmp_path / "download.bin"
assert stream.stream_response_to_file(response(), str(target)) == str(target)
assert target.read_bytes() == b"download"
supplied = BytesIO()
assert stream.stream_response_to_file(response(), supplied) is None
assert not supplied.closed
assert supplied.getvalue() == b"download"