From d68b0a146aa716b3e0fd83b86dc1585b2c3adcdc Mon Sep 17 00:00:00 2001 From: jayunit100 Date: Fri, 30 May 2014 06:23:28 -0400 Subject: [PATCH 01/14] fixing incorrect error message when vol parameter is missing --- .../java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java index 51ce5fee..fe02af81 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java @@ -114,8 +114,7 @@ public void setConf(Configuration conf){ String vol = conf.get("fs.glusterfs.volume.fuse." + v[i] , null); if(vol==null){ - log.error("Could not find property: fs.glusterfs.fuse." + v[i]); - throw new RuntimeException("Could not find mount point for volume: "+ v[i]); + throw new RuntimeException("Could not find property: fs.glusterfs.volume.fuse." + v[i] + "(mount point)"); } volumes.put(v[i],vol); log.info("Gluster volume: " + v[i] + " at : " + volumes.get(v[i])); From c03507833df3d235da0e8d545d36b221456dd9af Mon Sep 17 00:00:00 2001 From: jayunit100 Date: Fri, 30 May 2014 06:58:45 -0400 Subject: [PATCH 02/14] Another important log info message which clarifies failures in sameVolume() when mapreduce.jobtracker.system.dir is unqualified/nonexistent --- .../java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java index fe02af81..8d035582 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java @@ -125,6 +125,7 @@ public void setConf(Configuration conf){ }else{ attr = new GlusterFSXattr(); } + String jtSysDir = conf.get("mapreduce.jobtracker.system.dir", null); Path mapredSysDirectory = null; @@ -133,7 +134,7 @@ public void setConf(Configuration conf){ else{ mapredSysDirectory = new Path(conf.get("mapred.system.dir", "glusterfs:///mapred/system")); } - + log.info("Attempting to setup mapred sys directory: " + mapredSysDirectory); if(sameVolume(mapredSysDirectory) && !exists(mapredSysDirectory) ){ mkdirs(mapredSysDirectory); } From 1360963a7b6c2c1353ad1195295356d2b0384819 Mon Sep 17 00:00:00 2001 From: jayunit100 Date: Thu, 15 May 2014 14:43:08 -0400 Subject: [PATCH 03/14] Allow plugin to run without git properties missing --- src/main/java/org/apache/hadoop/fs/glusterfs/Version.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/Version.java b/src/main/java/org/apache/hadoop/fs/glusterfs/Version.java index 2c5a5aba..994cce37 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/Version.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/Version.java @@ -4,19 +4,24 @@ import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Versioning stuff for the shim. This class is not tested since there is no * deterministic behaviour (i.e. it might not work if not building from binary), * and the effects are pure side effects. */ public class Version extends Properties{ + static final Logger LOG = LoggerFactory.getLogger(Version.class); public Version() { super(); try{ load(this.getClass().getClassLoader().getResourceAsStream("git.properties")); } catch(Throwable t){ - throw new RuntimeException("Couldn't find git properties for version info " + t.getMessage()); + LOG.warn("Couldn't find GIT properties for version info " + + t.getMessage()+". This jar may have been built OUTSIDE a GIT repo."); } } public String getTag(){ From 7e3a0ee861e088270fe8ddce19bd3d2b9d468de0 Mon Sep 17 00:00:00 2001 From: jayunit100 Date: Thu, 15 May 2014 14:55:27 -0400 Subject: [PATCH 04/14] add simple unit test to confirm version works w/o git.properties in classpath --- src/main/java/org/apache/hadoop/fs/glusterfs/Version.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/Version.java b/src/main/java/org/apache/hadoop/fs/glusterfs/Version.java index 994cce37..31b0939b 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/Version.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/Version.java @@ -25,7 +25,11 @@ public Version() { } } public String getTag(){ - return this.getProperty("git.commit.id.describe").split("-")[0]; + String commit = this.getProperty("git.commit.id.describe"); + String tag = commit != null ? + commit.split("-")[0]: + "no version info available. check log warnings."; + return commit.split("-")[0]; } /** From bbe9395c4d6673411fd6b89d6e669de740679f34 Mon Sep 17 00:00:00 2001 From: rhbdjenkins Date: Fri, 30 May 2014 16:34:26 +0000 Subject: [PATCH 05/14] [ci-skip] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 758fbeba..25fe15e3 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ jar - 2.3.3 + 2.3.4 glusterfs-hadoop http://maven.apache.org From 2f9446b4740a9f268177eb634be1c25d0c08456f Mon Sep 17 00:00:00 2001 From: rhbdjenkins Date: Wed, 4 Jun 2014 21:14:04 +0000 Subject: [PATCH 06/14] [ci-skip] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 25fe15e3..f63c33ac 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ jar - 2.3.4 + 2.3.5 glusterfs-hadoop http://maven.apache.org From 2f00540ce2a105a51ab884c21487ec946109a1fe Mon Sep 17 00:00:00 2001 From: childsb Date: Mon, 9 Jun 2014 12:54:33 -0400 Subject: [PATCH 07/14] Update of the groupid to reflect sub-project of www.gluster.org --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f63c33ac..ae792228 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,7 @@ 4.0.0 - org.apache.hadoop.fs.glusterfs + org.gluster glusterfs-hadoop jar - 2.3.5 + 2.3.6 glusterfs-hadoop http://maven.apache.org From 09373e77b00c38c815994b34daaaa70200d1ce09 Mon Sep 17 00:00:00 2001 From: childsb Date: Wed, 25 Jun 2014 11:49:49 -0400 Subject: [PATCH 09/14] Throw RuntimeException if a volume doesn't exist. --- .../apache/hadoop/fs/glusterfs/GlusterVolume.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java index 8d035582..06d1f360 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java @@ -198,8 +198,12 @@ public File pathToFile(Path path) { }else if(volume==null){ volume = default_volume; } - - return new File(this.volumes.get(volume) + "/" + path.toUri().getPath()); + String volPath = this.volumes.get(volume); + if(volPath==null){ + throw new RuntimeException("Error undefined volume:" + volume + " in path: " + path); + } + + return new File(volPath + "/" + path.toUri().getPath()); } protected Path getInitialWorkingDirectory() { @@ -222,6 +226,10 @@ public Path fileToPath(File path) { root = nextPath; } } + + if(volume==null){ + throw new RuntimeException("No volume matching path: " + path); + } if(default_volume.equalsIgnoreCase(volume)) volume = ""; From 65dfb81d24860dae3017dc370d411f60a0c384d1 Mon Sep 17 00:00:00 2001 From: rhbdjenkins Date: Wed, 25 Jun 2014 16:28:37 +0000 Subject: [PATCH 10/14] [ci-skip] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 03ee3f3e..9c6395ba 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ jar - 2.3.6 + 2.3.7 glusterfs-hadoop http://maven.apache.org From 27f6529460c038ca2114f53801c562f16c661edf Mon Sep 17 00:00:00 2001 From: childsb Date: Thu, 26 Jun 2014 12:05:33 -0400 Subject: [PATCH 11/14] Throw a better error message if attempting to access a directory without permission. --- .../java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java index 06d1f360..dcf70e62 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java @@ -289,6 +289,10 @@ public FileStatus[] listStatus(Path f) throws IOException { return new FileStatus[] { new GlusterFileStatus(localf, getDefaultBlockSize(), this) }; } + + if(localf.isDirectory() && !localf.canRead()){ + throw new IOException("Access denied."); + } File[] names = localf.listFiles(); if (names == null) { From 68d5b254c48f63fc7a3421019c073198196ac81b Mon Sep 17 00:00:00 2001 From: childsb Date: Thu, 26 Jun 2014 13:18:18 -0400 Subject: [PATCH 12/14] Add path to error message. --- src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java index dcf70e62..135227e0 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java @@ -291,7 +291,7 @@ public FileStatus[] listStatus(Path f) throws IOException { } if(localf.isDirectory() && !localf.canRead()){ - throw new IOException("Access denied."); + throw new IOException("Access denied : " + localf.getPath()); } File[] names = localf.listFiles(); From 3c60812e8cd4b8bea3f7f545787259cde4db0dc2 Mon Sep 17 00:00:00 2001 From: rhbdjenkins Date: Thu, 26 Jun 2014 17:29:00 +0000 Subject: [PATCH 13/14] [ci-skip] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9c6395ba..146f00c6 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ jar - 2.3.7 + 2.3.8 glusterfs-hadoop http://maven.apache.org From b1d183d21e21b27c4dc587b5a8cc891a7718ac1b Mon Sep 17 00:00:00 2001 From: jayunit100 Date: Mon, 30 Jun 2014 09:38:02 -0400 Subject: [PATCH 14/14] Neccessary amend to pull #96 - still throws exceptions in some cases --- src/main/java/org/apache/hadoop/fs/glusterfs/Version.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/Version.java b/src/main/java/org/apache/hadoop/fs/glusterfs/Version.java index 31b0939b..90fe9f0e 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/Version.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/Version.java @@ -21,15 +21,16 @@ public Version() { } catch(Throwable t){ LOG.warn("Couldn't find GIT properties for version info " + - t.getMessage()+". This jar may have been built OUTSIDE a GIT repo."); + t +". This jar may have been built OUTSIDE a GIT repo."); } } public String getTag(){ String commit = this.getProperty("git.commit.id.describe"); - String tag = commit != null ? + final String tag = commit != null ? commit.split("-")[0]: "no version info available. check log warnings."; - return commit.split("-")[0]; + + return tag; } /**