Merge pull request #87 from bbockelm/crash_fixups

Fix deadlocks and crashes

Author: bbockelm

src/S3File.cc

@@ -547,6 +547,11 @@ S3File::DownloadBypass(off_t offset, size_t size, char *buffer) {
 	return std::make_tuple(-1, 0, true);
 }
 
+S3File::S3Cache::~S3Cache() {
+	std::unique_lock lk(m_mutex);
+	m_cv.wait(lk, [&] { return !m_a.m_inprogress && !m_b.m_inprogress; });
+}
+
 bool S3File::S3Cache::CouldUseAligned(off_t req, off_t cache) {
 	if (req < 0 || cache < 0) {
 		return false;
@@ -974,14 +979,24 @@ void S3File::S3Cache::Entry::Download(S3File &file) {
 	if (m_off + static_cast<off_t>(request_size) > file.content_length) {
 		request_size = file.content_length - m_off;
 	}
-	if (!m_request->SendRequest(m_off, m_cache_entry_size)) {
+	// This function is always called with m_mutex held; however,
+	// SendRequest can block if the threads are all busy; the threads
+	// will need to grab the lock to notify of completion.  So, we
+	// must release the lock here before calling a blocking function --
+	// otherwise deadlock may occur.
+	auto off = m_off;
+	m_parent.m_mutex.unlock();
+	if (!m_request->SendRequest(off, m_cache_entry_size)) {
+		m_parent.m_mutex.lock();
 		std::stringstream ss;
 		ss << "Failed to send GetObject command: "
 		   << m_request->getResponseCode() << "'"
 		   << m_request->getResultString() << "'";
 		file.m_log.Log(LogMask::Warning, "S3File::Read", ss.str().c_str());
 		m_failed = true;
 		m_request.reset();
+	} else {
+		m_parent.m_mutex.lock();
 	}
 }
 

src/S3File.hh

@@ -291,6 +291,10 @@ class S3File : public XrdOssDF {
 
 		// Trigger a blocking read from a given file
 		ssize_t Read(S3File &file, char *buffer, off_t offset, size_t size);
+
+		// Shutdown the cache; ensure all reads are completed before
+		// deleting the objects.
+		~S3Cache();
 	};
 	S3Cache m_cache;
 };

test/s3-setup.sh

@@ -204,7 +204,7 @@ echo "Hello, World" > "$RUNDIR/hello_world.txt"
 IDX=0
 COUNT=25
 while [ $IDX -ne $COUNT ]; do
-  if ! dd if=/dev/urandom "of=$RUNDIR/test_file" bs=1024 count=1024 2> /dev/null; then
+  if ! dd if=/dev/urandom "of=$RUNDIR/test_file" bs=1024 count=3096 2> /dev/null; then
     echo "Failed to create random file to upload"
     exit 1
   fi