Merge pull request #107 from rw2/http-ls

Http ls

Author: bbockelm

CMakeLists.txt

@@ -112,11 +112,12 @@ target_link_libraries(XrdOssS3 XrdS3Obj)
 ######################
 ##  libXrdOssHTTP   ##
 ######################
-add_library(XrdHTTPServerObj OBJECT src/CurlUtil.cc src/HTTPFile.cc src/HTTPFileSystem.cc src/HTTPCommands.cc src/TokenFile.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
+add_library(XrdHTTPServerObj OBJECT src/CurlUtil.cc src/HTTPFile.cc src/HTTPDirectory.cc src/HTTPFileSystem.cc src/HTTPCommands.cc src/TokenFile.cc src/stl_string_utils.cc src/shortfile.cc src/logging.cc)
 set_target_properties(XrdHTTPServerObj PROPERTIES POSITION_INDEPENDENT_CODE ON)
 target_include_directories(XrdHTTPServerObj PRIVATE ${XRootD_INCLUDE_DIRS})
-target_link_libraries(XrdHTTPServerObj ${XRootD_UTILS_LIBRARIES} ${XRootD_SERVER_LIBRARIES} CURL::libcurl OpenSSL::Crypto Threads::Threads std::filesystem)
+target_link_libraries(XrdHTTPServerObj ${XRootD_UTILS_LIBRARIES} ${XRootD_SERVER_LIBRARIES} CURL::libcurl  tinyxml2::tinyxml2 OpenSSL::Crypto Threads::Threads std::filesystem)
 
+# Compatability library, doesn't match upstream's naming convention
 add_library(XrdHTTPServer MODULE "$<TARGET_OBJECTS:XrdHTTPServerObj>")
 target_link_libraries(XrdHTTPServer XrdHTTPServerObj)
 # New library name, matches upstream's naming convention

src/GlobusDirectory.hh

@@ -19,22 +19,25 @@
 #pragma once
 
 #include "GlobusFileSystem.hh"
-#include "HTTPDirectory.hh"
+
+#include <XrdOss/XrdOss.hh>
 
 #include <string>
 #include <vector>
 
+class XrdSysError;
+
 // Structure to hold Globus object information
 struct GlobusObjectInfo {
 	size_t m_size;
 	std::string m_key;
 	std::string m_last_modified;
 };
 
-class GlobusDirectory : public HTTPDirectory {
+class GlobusDirectory : public XrdOssDF {
   public:
 	GlobusDirectory(XrdSysError &log, const GlobusFileSystem &fs)
-		: HTTPDirectory(log), m_fs(fs) {}
+		: m_log(log), m_fs(fs) {}
 
 	virtual ~GlobusDirectory() {}
 
@@ -56,6 +59,7 @@ class GlobusDirectory : public HTTPDirectory {
 	std::vector<GlobusObjectInfo> m_directories;
 	std::string m_prefix;
 	std::string m_object;
+	XrdSysError &m_log;
 	const GlobusFileSystem &m_fs;
 	struct stat *m_stat_buf{nullptr};
 };

src/HTTPCommands.cc

@@ -873,6 +873,18 @@ bool HTTPDownload::SendRequest(off_t offset, size_t size) {
 
 // ---------------------------------------------------------------------------
 
+HTTPList::~HTTPList() {}
+
+bool HTTPList::SendRequest() {
+	expectedResponseCode = {200};
+
+	httpVerb = "GET";
+	std::string noPayloadAllowed;
+	return SendHTTPRequest(noPayloadAllowed);
+}
+
+// ---------------------------------------------------------------------------
+
 HTTPHead::~HTTPHead() {}
 
 bool HTTPHead::SendRequest() {

src/HTTPCommands.hh

@@ -351,6 +351,22 @@ class HTTPDownload final : public HTTPRequest {
 	std::string object;
 };
 
+class HTTPList final : public HTTPRequest {
+  public:
+	HTTPList(const std::string &h, const std::string &o, XrdSysError &log,
+			 const TokenFile *token)
+		: HTTPRequest(h, log, token), object(o) {
+		hostUrl = hostUrl + "/" + object;
+	}
+
+	virtual ~HTTPList();
+
+	virtual bool SendRequest();
+
+  protected:
+	std::string object;
+};
+
 class HTTPHead final : public HTTPRequest {
   public:
 	HTTPHead(const std::string &h, const std::string &o, XrdSysError &log,

src/HTTPDirectory.cc

@@ -0,0 +1,204 @@
+/***************************************************************
+ *
+ * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.  You may
+ * obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ ***************************************************************/
+
+#include "HTTPDirectory.hh"
+#include "HTTPCommands.hh"
+#include "HTTPFile.hh"
+#include "HTTPFileSystem.hh"
+#include "logging.hh"
+#include "stl_string_utils.hh"
+
+#include <XrdOuc/XrdOucEnv.hh>
+#include <XrdOuc/XrdOucStream.hh>
+#include <XrdSec/XrdSecEntity.hh>
+#include <XrdSec/XrdSecEntityAttr.hh>
+#include <XrdSfs/XrdSfsInterface.hh>
+#include <XrdSys/XrdSysError.hh>
+#include <XrdVersion.hh>
+
+#include <curl/curl.h>
+#include <filesystem>
+#include <iostream>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <regex>
+#include <sstream>
+#include <string>
+#include <tinyxml2.h>
+
+using namespace XrdHTTPServer;
+
+HTTPDirectory::HTTPDirectory(XrdSysError &log, HTTPFileSystem &oss)
+	: m_log(log), m_oss(oss) {} // Initialize it to false.
+
+void HTTPDirectory::parseHTMLToListing(const std::string &htmlContent) {
+	m_remoteList.clear();
+
+	tinyxml2::XMLDocument doc;
+	tinyxml2::XMLError error = doc.Parse(htmlContent.c_str());
+	if (error != tinyxml2::XML_SUCCESS) {
+		m_log.Log(LogMask::Warning, "HTTPDirectory",
+				  "Failed to parse HTML in directory response");
+		return;
+	}
+
+	// Root of the HTML document
+	auto root = doc.FirstChild();
+	if (!root) {
+		m_log.Log(LogMask::Warning, "HTTPDirectory",
+				  "No root found in HTML in directory response");
+		return;
+	}
+
+	// Traverse the rows in the table
+	for (auto row = root->FirstChildElement("tr"); row != nullptr;
+		 row = row->NextSiblingElement("tr")) {
+		Entry entry;
+		int columnIndex = 0;
+
+		// Traverse each cell in the row
+		for (auto cell = row->FirstChildElement("td"); cell != nullptr;
+			 cell = cell->NextSiblingElement("td")) {
+			const char *cellText = cell->GetText() ? cell->GetText() : "";
+
+			switch (columnIndex) {
+			case 0: // Mode
+				entry.mode = cellText;
+				break;
+			case 1: // Flags
+				entry.flags = cellText;
+				break;
+			case 2: // Size
+				entry.size = cellText;
+				break;
+			case 3: // Modified
+				entry.modified = cellText;
+				break;
+			case 4: // Name
+				if (auto aTag = cell->FirstChildElement("a")) {
+					const char *nameText =
+						aTag->GetText() ? aTag->GetText() : "";
+					entry.name = nameText;
+				}
+				break;
+			default:
+				break;
+			}
+			columnIndex++;
+		}
+
+		// Skip adding invalid/empty rows
+		if (entry.name.empty()) {
+			continue;
+		}
+
+		struct stat workingFile;
+		workingFile.st_size = std::stoul(entry.size, nullptr, 10);
+		// workingFile.st_mtime = std::stoul(entry.modified, nullptr, 10);
+		if (entry.mode.substr(0, 1) == "d")
+			workingFile.st_mode = 0600 | S_IFDIR;
+		else
+			workingFile.st_mode = 0600 | S_IFREG;
+
+		workingFile.st_nlink = 1;
+		workingFile.st_uid = 1;
+		workingFile.st_gid = 1;
+		workingFile.st_atime = 0;
+		workingFile.st_ctime = 0;
+		workingFile.st_dev = 0;
+		workingFile.st_ino = 0;
+		m_remoteList.push_back({entry.name, workingFile});
+	}
+}
+
+std::string HTTPDirectory::extractHTMLTable(const std::string &htmlContent) {
+	std::regex tableRegex(R"(<table[^>]*>[\s\S]*?</table>)",
+						  std::regex_constants::icase);
+
+	std::smatch match;
+	if (std::regex_search(htmlContent, match, tableRegex)) {
+		return match.str();
+	}
+
+	return ""; // Return an empty string if no table is found
+}
+
+int HTTPDirectory::Readdir(char *buff, int blen) {
+	if (m_remoteList.size() > 0) {
+		std::string name = m_remoteList.begin()->first;
+		struct stat currentRecord = m_remoteList.begin()->second;
+		mystat->st_size = currentRecord.st_size;
+		mystat->st_mode = currentRecord.st_mode;
+		mystat->st_nlink = currentRecord.st_nlink;
+		mystat->st_uid = currentRecord.st_uid;
+		mystat->st_gid = currentRecord.st_gid;
+		mystat->st_atime = currentRecord.st_atime;
+		mystat->st_ctime = currentRecord.st_ctime;
+		mystat->st_dev = currentRecord.st_dev;
+		mystat->st_ino = currentRecord.st_ino;
+		memcpy(buff, name.c_str(), name.size() + 1);
+		m_remoteList.erase(m_remoteList.begin());
+		return name.size();
+	} else {
+		buff[0] = '\0';
+		return 0;
+	}
+}
+
+int HTTPDirectory::Opendir(const char *path, XrdOucEnv &env) {
+	m_log.Log(LogMask::Debug, "HTTPDirectory::Opendir", "Opendir called");
+	auto configured_hostname = m_oss.getHTTPHostName();
+	auto configured_hostUrl = m_oss.getHTTPHostUrl();
+	const auto &configured_url_base = m_oss.getHTTPUrlBase();
+	if (!configured_url_base.empty()) {
+		configured_hostUrl = configured_url_base;
+		configured_hostname = m_oss.getStoragePrefix();
+	}
+
+	//
+	// Check the path for validity.
+	//
+	std::string object;
+	int rv = parse_path(m_oss.getHTTPHostName(), path, object);
+
+	if (rv != 0) {
+		return rv;
+	}
+
+	if (m_remoteList.empty()) {
+		m_log.Log(LogMask::Debug, "HTTPFile::Opendir", "Opendir called");
+		HTTPList list(configured_hostUrl, object, m_log, m_oss.getToken());
+		m_log.Log(LogMask::Debug, "HTTPDirectory::Opendir",
+				  "About to perform download from HTTPDirectory::Opendir(): "
+				  "hostname / object:",
+				  configured_hostname.c_str(), object.c_str());
+		if (!list.SendRequest()) {
+			std::stringstream ss;
+			ss << "Failed to send GetObject command: " << list.getResponseCode()
+			   << "'" << list.getResultString() << "'";
+			m_log.Log(LogMask::Warning, "HTTPDirectory::Opendir",
+					  ss.str().c_str());
+			return 0;
+		}
+
+		parseHTMLToListing(extractHTMLTable(list.getResultString()));
+	}
+
+	return 0;
+}

src/HTTPDirectory.hh

@@ -18,27 +18,46 @@
 
 #pragma once
 
-#include "XrdOss/XrdOss.hh"
-#include "XrdOuc/XrdOucEnv.hh"
+#include "HTTPFileSystem.hh"
+#include "logging.hh"
 
-class XrdSysError;
+#include <XrdOss/XrdOss.hh>
+#include <XrdOuc/XrdOucEnv.hh>
+#include <XrdSfs/XrdSfsInterface.hh>
+
+#include <map>
+#include <vector>
 
 class HTTPDirectory : public XrdOssDF {
   public:
-	HTTPDirectory(XrdSysError &log) : m_log(log) {}
-
+	HTTPDirectory(XrdSysError &log, HTTPFileSystem &oss);
 	virtual ~HTTPDirectory() {}
 
-	virtual int Opendir(const char *path, XrdOucEnv &env) override {
-		return -ENOSYS;
-	}
+	virtual int Opendir(const char *path, XrdOucEnv &env) override;
 
-	virtual int Readdir(char *buff, int blen) override { return -ENOSYS; }
+	virtual int Readdir(char *buff, int blen) override;
 
-	virtual int StatRet(struct stat *statStruct) override { return -ENOSYS; }
+	virtual int StatRet(struct stat *statStruct) override {
+		mystat = statStruct;
+		return SFS_OK;
+	}
 
 	virtual int Close(long long *retsz = 0) override { return -ENOSYS; }
 
-  protected:
+  private:
+	struct Entry {
+		std::string mode;
+		std::string flags;
+		std::string size;
+		std::string modified;
+		std::string name;
+	};
+
+	void parseHTMLToListing(const std::string &htmlContent);
+	std::string extractHTMLTable(const std::string &htmlContent);
+
+	struct stat *mystat;
 	XrdSysError &m_log;
+	HTTPFileSystem &m_oss;
+	std::vector<std::pair<std::string, struct stat>> m_remoteList;
 };