diff --git a/requests_toolbelt/downloadutils/stream.py b/requests_toolbelt/downloadutils/stream.py index 7253d96e..f90854d8 100644 --- a/requests_toolbelt/downloadutils/stream.py +++ b/requests_toolbelt/downloadutils/stream.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """Utilities for dealing with streamed requests.""" +import errno import os.path import re @@ -13,6 +14,9 @@ _QUOTED_STRING_RE) ) _DEFAULT_CHUNKSIZE = 512 +# Create the file only if it does not exist yet, as one atomic step. +_EXCLUSIVE_CREATE_FLAGS = (os.O_WRONLY | os.O_CREAT | os.O_EXCL | + getattr(os, 'O_BINARY', 0)) def _get_filename(content_disposition): @@ -163,9 +167,17 @@ def stream_response_to_file(response, path=None, chunksize=_DEFAULT_CHUNKSIZE): filename = getattr(fd, 'name', None) else: filename = get_download_file_path(response, path) - if os.path.exists(filename): + try: + descriptor = os.open(filename, _EXCLUSIVE_CREATE_FLAGS, 0o666) + except OSError as error: + if error.errno != errno.EEXIST: + raise raise exc.StreamingError("File already exists: %s" % filename) - fd = open(filename, 'wb') + try: + fd = os.fdopen(descriptor, 'wb') + except BaseException: + os.close(descriptor) + raise for chunk in response.iter_content(chunk_size=chunksize): fd.write(chunk) diff --git a/tests/test_download_creation_race.py b/tests/test_download_creation_race.py new file mode 100644 index 00000000..99b467c2 --- /dev/null +++ b/tests/test_download_creation_race.py @@ -0,0 +1,71 @@ +"""A competing creator must never lose its existing bytes.""" +import os +from io import BytesIO + +try: + from unittest.mock import Mock +except ImportError: + from mock import Mock + +import pytest + +from requests_toolbelt.downloadutils import stream +from requests_toolbelt.exceptions import StreamingError + + +def response(): + result = Mock() + result.iter_content.return_value = [b"download"] + return result + + +def test_competing_file_creation_preserves_winner(tmp_path, monkeypatch): + target = tmp_path / "download.bin" + real_os_open = os.open + + def race(path, flags, mode=0o777): + with open(path, "wb") as winner: + winner.write(b"competing file") + return real_os_open(path, flags, mode) + + monkeypatch.setattr(stream.os, "open", race) + with pytest.raises(StreamingError, match="File already exists"): + stream.stream_response_to_file(response(), str(target)) + assert target.read_bytes() == b"competing file" + + +def test_descriptor_is_closed_when_wrapping_fails(tmp_path, monkeypatch): + target = tmp_path / "download.bin" + real_os_open = os.open + real_os_close = os.close + opened = [] + closed = [] + + def record_open(path, flags, mode=0o777): + opened.append(real_os_open(path, flags, mode)) + return opened[-1] + + def record_close(descriptor): + closed.append(descriptor) + real_os_close(descriptor) + + def fail_fdopen(descriptor, mode): + raise MemoryError("synthetic fdopen failure") + + monkeypatch.setattr(stream.os, "open", record_open) + monkeypatch.setattr(stream.os, "close", record_close) + monkeypatch.setattr(stream.os, "fdopen", fail_fdopen) + with pytest.raises(MemoryError, match="synthetic fdopen failure"): + stream.stream_response_to_file(response(), str(target)) + assert len(opened) == 1 + assert closed == opened + + +def test_new_file_and_caller_owned_stream(tmp_path): + target = tmp_path / "download.bin" + assert stream.stream_response_to_file(response(), str(target)) == str(target) + assert target.read_bytes() == b"download" + supplied = BytesIO() + assert stream.stream_response_to_file(response(), supplied) is None + assert not supplied.closed + assert supplied.getvalue() == b"download"