Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions fsspec/implementations/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -522,6 +522,30 @@ async def _isdir(self, path):
except (FileNotFoundError, ValueError):
return False

async def _pipe_file(self, path, value, mode="overwrite", **kwargs):
"""
Write bytes to a remote file over HTTP.

Parameters
----------
path : str
Target URL where the data should be written
value : bytes
Data to be written
mode : str
How to write to the file - 'overwrite' or 'append'
**kwargs : dict
Additional parameters to pass to the HTTP request
"""
url = self._strip_protocol(path)
headers = kwargs.pop("headers", {})
headers["Content-Length"] = str(len(value))

session = await self.get_session()

async with session.put(url, data=value, headers=headers, **kwargs) as r:
r.raise_for_status()


class HTTPFile(AbstractBufferedFile):
"""
Expand Down
40 changes: 40 additions & 0 deletions fsspec/implementations/tests/test_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -581,3 +581,43 @@ async def test_async_walk(server):
pass

await fs._session.close()


def test_pipe_file(server, tmpdir, reset_files):
"""Test that the pipe_file method works correctly."""
import io

import fsspec

# Create test data
test_content = b"This is test data to pipe to a file"

# Initialize filesystem
fs = fsspec.filesystem("http", headers={"accept_put": "true"})

# Test that the file doesn't exist yet
with pytest.raises(FileNotFoundError):
fs.info(server.address + "/piped_file")

# Pipe data to the file
fs.pipe_file(server.address + "/piped_file", test_content)

# Verify the file exists now
assert fs.exists(server.address + "/piped_file")

# Verify content
assert fs.cat(server.address + "/piped_file") == test_content

# Test with different modes and headers
fs.pipe_file(
server.address + "/piped_file2",
test_content,
mode="overwrite",
headers={"Content-Type": "text/plain"},
)
assert fs.cat(server.address + "/piped_file2") == test_content

# Test with byte-like object
bytesio = io.BytesIO(b"BytesIO content")
fs.pipe_file(server.address + "/piped_bytes", bytesio.getvalue())
assert fs.cat(server.address + "/piped_bytes") == b"BytesIO content"