Implement created and modified methods from abstract filesystem class… (#1971)

Author: dsevilla

fsspec/implementations/tests/test_webhdfs.py

@@ -2,6 +2,7 @@
 import shlex
 import subprocess
 import time
+from datetime import datetime
 
 import pytest
 
@@ -208,3 +209,87 @@ def test_protocol_prefixed_path(hdfs_cluster):
 
     file_info = fs.ls(protocol_prefixed_path, detail=True)
     assert len(file_info) == 0
+
+
+def test_modified_nonexistent_path(hdfs_cluster):
+    fs = WebHDFS(
+        hdfs_cluster,
+        user="testuser",
+        data_proxy={"worker.example.com": "localhost"},
+    )
+    nonexistent_path = "/user/testuser/nonexistent_file.txt"
+
+    with pytest.raises(FileNotFoundError):
+        fs.modified(nonexistent_path)
+
+
+def test_modified_time(hdfs_cluster):
+    fs = WebHDFS(
+        hdfs_cluster,
+        user="testuser",
+        data_proxy={"worker.example.com": "localhost"},
+    )
+    dir_path = "/user/testuser/"
+    file_path = f"{dir_path}/testfile.txt"
+
+    fs.mkdir(dir_path)
+
+    # Check first modified time for directories
+    modified_dir_date: datetime = fs.modified(dir_path)
+
+    # I think it is the only thing we can assume, but I'm not sure if the server has a different time
+    assert modified_dir_date <= datetime.now()
+
+    # Create a file and check modified time again
+    with fs.open(file_path, "wb") as f:
+        f.write(b"test content")
+
+    modified_file_date: datetime = fs.modified(file_path)
+    assert modified_file_date >= modified_dir_date
+    assert modified_file_date <= datetime.now()
+
+
+# NOTE: These following two tests are a copy of the modified ones, as
+# WebHDFS does not have a created time API, we are using modified as a proxy.
+
+
+def test_created_nonexistent_path(hdfs_cluster):
+    fs = WebHDFS(
+        hdfs_cluster,
+        user="testuser",
+        data_proxy={"worker.example.com": "localhost"},
+    )
+    nonexistent_path = "/user/testuser/nonexistent_file.txt"
+
+    with pytest.raises(FileNotFoundError):
+        fs.created(nonexistent_path)
+
+
+def test_created_time(hdfs_cluster):
+    fs = WebHDFS(
+        hdfs_cluster,
+        user="testuser",
+        data_proxy={"worker.example.com": "localhost"},
+    )
+    dir_path = "/user/testuser/"
+    file_path = f"{dir_path}/testfile.txt"
+
+    fs.mkdir(dir_path)
+
+    time.sleep(1)
+
+    # Check first created time for directories
+    created_dir_date: datetime = fs.created(dir_path)
+
+    # I think it is the only thing we can assume, but I'm not sure if the server has a different time
+    assert created_dir_date < datetime.now()
+
+    # Create a file and check created time again
+    with fs.open(file_path, "wb") as f:
+        f.write(b"test content")
+
+    time.sleep(1)
+
+    created_file_date: datetime = fs.created(file_path)
+    assert created_file_date > created_dir_date
+    assert created_file_date < datetime.now()

fsspec/implementations/webhdfs.py

@@ -7,6 +7,7 @@
 import tempfile
 import uuid
 from contextlib import suppress
+from datetime import datetime
 from urllib.parse import quote
 
 import requests
@@ -268,6 +269,23 @@ def info(self, path):
         info["name"] = path
         return self._process_info(info)
 
+    def created(self, path):
+        """Return the created timestamp of a file as a datetime.datetime"""
+        # The API does not provide creation time, so we use modification time
+        info = self.info(path)
+        mtime = info.get("modificationTime", None)
+        if mtime is not None:
+            return datetime.fromtimestamp(mtime / 1000)
+        raise RuntimeError("Could not retrieve creation time (modification time).")
+
+    def modified(self, path):
+        """Return the modified timestamp of a file as a datetime.datetime"""
+        info = self.info(path)
+        mtime = info.get("modificationTime", None)
+        if mtime is not None:
+            return datetime.fromtimestamp(mtime / 1000)
+        raise RuntimeError("Could not retrieve modification time.")
+
     def ls(self, path, detail=False, **kwargs):
         out = self._call("LISTSTATUS", path=path)
         infos = out.json()["FileStatuses"]["FileStatus"]