From 77592b64eed839e000f725fd97d5a23bdc33b12d Mon Sep 17 00:00:00 2001 From: jayunit100 Date: Mon, 12 Aug 2013 18:53:05 -0400 Subject: [PATCH 1/3] Lenient Check path implementation that confirms the scheme and returns - avoids complex logic of RawLocalFileSystem. --- .../fs/glusterfs/GlusterFileSystem.java | 5 ++++ .../fs/glusterfs/GlusterFileSystemCRC.java | 6 ++++ .../hadoop/fs/glusterfs/GlusterVolume.java | 12 ++++++++ .../org/apache/hadoop/fs/glusterfs/Util.java | 30 +++++++++++++++++++ 4 files changed, 53 insertions(+) diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java index 8107f862..2c698620 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java @@ -113,6 +113,11 @@ public void copyToLocalFile(boolean delSrc,Path src,Path dst) throws IOException FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf()); } + @Override + protected void checkPath(Path path){ + Util.checkPath(this, path); + } + public String toString(){ return "Gluster File System, no CRC."; } diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystemCRC.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystemCRC.java index 2739a77d..84944222 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystemCRC.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystemCRC.java @@ -71,6 +71,12 @@ public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOExcepti FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf()); } + + @Override + protected void checkPath(Path path){ + Util.checkPath(this, path); + } + public String toString(){ return "Gluster File System - CRC Enabled"; } diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java index ea2699cf..e705a904 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java @@ -160,7 +160,19 @@ public BlockLocation[] getFileBlockLocations(FileStatus file,long start,long len return result; } + + @Override + protected void checkPath(Path path){ + Util.checkPath(this, path); + } + /** + * May revert to include this at some point. + * protected void checkPath(Path arg0){ + * super.checkPath(arg0); + * } + */ + public String toString(){ return "Gluster Volume mounted at: " + root; } diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/Util.java b/src/main/java/org/apache/hadoop/fs/glusterfs/Util.java index 6c26be5b..26ebafba 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/Util.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/Util.java @@ -26,7 +26,9 @@ import java.io.File; import java.io.IOException; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Shell; public class Util{ @@ -50,4 +52,32 @@ public static String[] getGET_PERMISSION_COMMAND(){ public static final boolean WINDOWS /* borrowed from Path.WINDOWS */ =System.getProperty("os.name").startsWith("Windows"); // / loads permissions, owner, and group from `ls -ld` + + /** + * Check that a Path belongs to this FileSystem. + * lenient : doesn't check authority. This might be temporary ~ + * there could be a better long term implementation. + * */ + public static void checkPath(FileSystem fs, Path path) { + String thisScheme = fs.getUri().getScheme(); + String thisAuthority = fs.getUri().getAuthority(); + + String thatScheme = path.toUri().getScheme(); + String thatAuthority = path.toUri().getAuthority(); + + //String debugInfo="GV: checking path " +path+ " scheme=" + thisScheme+" auth="+thisAuthority + " vs scheme=" + thatScheme +" auth=" + thatAuthority; + //log.info(debugInfo); + //log.warn("Not validating authority"); + //now the exception will be traceable in the logs above . + try{ + //super.checkPath(path); + if(thisScheme.equals(thatScheme) || (thatScheme==null && thatAuthority==null)) + return ; + else + throw new RuntimeException("Schemes dont match: expecting :" + thisScheme + " but input path is :" + thatScheme); + } + catch(Throwable t){ + throw new RuntimeException("ERROR matching schemes/auths: " + thisScheme +" ~ " + thisAuthority + " : " + thatScheme + " ~ " + thatAuthority ); + } + } } From 7a58c5158c30d0bd311b38e6a9e3db68731cbf49 Mon Sep 17 00:00:00 2001 From: jayunit100 Date: Fri, 16 Aug 2013 11:03:16 -0400 Subject: [PATCH 2/3] cleanup and debug --- .../org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java | 7 +++++-- .../java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java | 7 +++++-- src/main/java/org/apache/hadoop/fs/glusterfs/Util.java | 4 +++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java index 2c698620..0dbffe8c 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java @@ -67,8 +67,11 @@ public GlusterFileSystem(){ log.info("GIT INFO="+v); log.info("GIT_TAG="+v.getTag()); } - - public GlusterFileSystem(FileSystem rawLocalFileSystem){ + + /** + * An internal constructor for creating FlusterFileSystem from a rawlocal fs. + */ + private GlusterFileSystem(FileSystem rawLocalFileSystem){ super(rawLocalFileSystem); rfs=rawLocalFileSystem; } diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java index e705a904..010136ad 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java @@ -35,6 +35,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * + */ public class GlusterVolume extends RawLocalFileSystem{ static final Logger log = LoggerFactory.getLogger(GlusterFileSystemCRC.class); @@ -53,7 +56,7 @@ public GlusterVolume(Configuration conf){ public URI getUri() { return NAME; } public void setConf(Configuration conf){ - log.info("Initializing gluster volume.."); + log.info("Initializing gluster volume: " + conf.toString()); super.setConf(conf); String getfattrcmd = null; if(conf!=null){ @@ -154,7 +157,7 @@ public BlockLocation[] getFileBlockLocations(FileStatus file,long start,long len result=attr.getPathInfo(f.getPath(), start, len); if(result==null){ - log.info("Problem getting destination host for file "+f.getPath()); + log.info("Problem getting destination host for file "+f.getPath() + " start=" + start + " len=" + len) ; return null; } diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/Util.java b/src/main/java/org/apache/hadoop/fs/glusterfs/Util.java index 26ebafba..9ea36c03 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/Util.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/Util.java @@ -56,7 +56,9 @@ public static String[] getGET_PERMISSION_COMMAND(){ /** * Check that a Path belongs to this FileSystem. * lenient : doesn't check authority. This might be temporary ~ - * there could be a better long term implementation. + * there could be a better long term implementation. + * Having custom implementation here is critical for debugging, because + * existing checkPath in hadoop doesn't print all the scheme/auth values. * */ public static void checkPath(FileSystem fs, Path path) { String thisScheme = fs.getUri().getScheme(); From 4b199385a4915e3d1597878a577011b6af3255c8 Mon Sep 17 00:00:00 2001 From: jayunit100 Date: Fri, 16 Aug 2013 11:27:23 -0400 Subject: [PATCH 3/3] protected constructor instead of private --- .../org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java index 0dbffe8c..662f6982 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java @@ -70,8 +70,12 @@ public GlusterFileSystem(){ /** * An internal constructor for creating FlusterFileSystem from a rawlocal fs. + * Example usage: + * + * FileSystem fs = new GlusterFileSystem(new GlusterVolume()); + * */ - private GlusterFileSystem(FileSystem rawLocalFileSystem){ + protected GlusterFileSystem(FileSystem rawLocalFileSystem){ super(rawLocalFileSystem); rfs=rawLocalFileSystem; }