From 989a183ff5aee5724353298bcb53a9b43e11c8f2 Mon Sep 17 00:00:00 2001 From: Ekasit Kijsipongse Date: Tue, 7 Jul 2015 10:43:29 +0700 Subject: [PATCH 1/2] Support pathinfo from gluster stripe volume --- .../fs/glusterfs/GlusterFSPathInfo.java | 263 ++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSPathInfo.java diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSPathInfo.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSPathInfo.java new file mode 100644 index 00000000..a6975b7a --- /dev/null +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSPathInfo.java @@ -0,0 +1,263 @@ +/** + * + * Copyright (c) 2011 Gluster, Inc. + * This file is part of GlusterFS. + * + * Licensed under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package org.apache.hadoop.fs.glusterfs; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; +import java.util.regex.Matcher; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.Path; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GlusterFSPathInfo { + static final Logger log = LoggerFactory.getLogger(GlusterFSPathInfo.class); + + + private static ConcurrentMap pathInfoCaches = new ConcurrentHashMap(); // The key is the filename, assuming that the file doesn't change during the execution time. + + private String pathInfo; + private long stripeSize; + private List> locations; + + public static final GlusterFSPathInfo INVALID = new GlusterFSPathInfo("",-1,null); + + public GlusterFSPathInfo(String pathInfo, long stripeSize, List> locations) { + this.pathInfo = pathInfo; + this.stripeSize = stripeSize; + this.locations = locations; + } + + /* + * @param pathInfo A string with no leading white spaces + * @return The offset of the next subvolume or brick (skip white spaces) + */ + private static int nextStripeOffset(String pathInfo) { + int len = pathInfo.length(); + int endIdx = 0; + if (pathInfo.charAt(0) == '<') { + endIdx = pathInfo.indexOf('>'); + } else if (pathInfo.charAt(0) == '(') { + int count = 1; + for (endIdx=1; endIdx < len; endIdx++) { + if (pathInfo.charAt(endIdx) == '(') + count++; + else if (pathInfo.charAt(endIdx) == ')') + count--; + if (count <= 0) + break; + } + if (count != 0) + throw new RuntimeException("Mismatched parenthesis in " + pathInfo); + } + + if (endIdx == 0) + throw new RuntimeException("Expected pattern not found " + pathInfo); + + // Skip trailing white spaces + for (endIdx++; endIdx < len; endIdx++) { + if (!Character.isWhitespace(pathInfo.charAt(endIdx))) + break; + } + + return endIdx; + } + + /* + * @param pathInfo A string starting with '(' + * @return The string between (excluding) outer parenthesis + */ + private static String getSubVolume(String pathInfo) { + int count = 0; + int idx=0; + for (; idx < pathInfo.length(); idx++) { + if (pathInfo.charAt(idx) == '(') + count++; + else if (pathInfo.charAt(idx) == ')') + count--; + if (count <= 0) + break; + } + + if (count == 0) + return pathInfo.substring(1,idx); + else + throw new RuntimeException("Mismatched parenthesis in " + pathInfo); + } + + /* + * @param pathInfo A glusterfs pathinfo extended attribute + * @return A GlusterFSPathInfo + */ + private static GlusterFSPathInfo parse(String pathInfo) { + String save = pathInfo; + + long stripeSize = Long.MAX_VALUE; + + Pattern pathInfoPattern = Pattern.compile("trusted.glusterfs.pathinfo=\"(.+)\"$"); + Pattern dhtPattern = Pattern.compile("^\\(]+> (.+)\\)$"); + Pattern stripePattern = Pattern.compile("^ (.+)$"); + Pattern replicatePattern = Pattern.compile("^]+> (.+)$"); + Pattern brickPattern = Pattern.compile(""); + + Matcher pathInfoMatcher = pathInfoPattern.matcher(pathInfo); + if (pathInfoMatcher.find()) { + pathInfo = pathInfoMatcher.group(1); + } else { + throw new RuntimeException("Cannot find pathinfo attribute"); + } + + // First level = DHT (exactly once) + Matcher dhtMatcher = dhtPattern.matcher(pathInfo); + if (dhtMatcher.find()) { + pathInfo = dhtMatcher.group(1); + } + + // Next level can be nested volume enclosed in () + if (pathInfo.startsWith("(")) { + String subVolume = getSubVolume(pathInfo); + if (subVolume.length() != pathInfo.length()-2) + log.warn("Ignore garbage at the end of pathInfo: " + pathInfo.substring(subVolume.length()+2)); // Should not have anything else since DHT must return 1 subvolume + + pathInfo = subVolume; + } + + // Check if STRIPE, then find the stripe index + int stripeIndex = 0; + int numStripes = 0; + List stripes = new ArrayList(); + Matcher stripeMatcher = stripePattern.matcher(pathInfo); + if (stripeMatcher.find()) { + stripeSize = Long.parseLong(stripeMatcher.group(1)); + log.debug("Stripe Size: " + stripeSize); + pathInfo = stripeMatcher.group(2); + + while (!pathInfo.isEmpty()) { + int nextOffset = nextStripeOffset(pathInfo); + if (pathInfo.charAt(0) == '(') { + stripes.add(pathInfo.substring(1,pathInfo.lastIndexOf(')',nextOffset))); + } else { + stripes.add(pathInfo.substring(0,nextOffset).trim()); + } + pathInfo = pathInfo.substring(nextOffset); + numStripes++; + } + + } else { + numStripes = 1; + stripes.add(pathInfo); // Pretending 1 stripe + } + + List> locations = new ArrayList>(); + for (String stripe: stripes) { + // Then, check if REPLICATE + pathInfo = stripe; + Matcher replicateMatcher = replicatePattern.matcher(pathInfo); + if (replicateMatcher.find()) { + pathInfo = replicateMatcher.group(1); + } + + // Last level = POSIX (mandatory) + Matcher brickMatcher = brickPattern.matcher(pathInfo); + List hosts = new ArrayList(); + while (brickMatcher.find()) { + hosts.add(brickMatcher.group(1)); + } + locations.add(hosts); + } + return new GlusterFSPathInfo(save, stripeSize, locations); + } + + /* + * @param filename A fuse-mounted filename + * @return A GlusterFSPathInfo + */ + public static GlusterFSPathInfo get(String filename) { + String cmdOut = new GlusterFSXattr(filename).getPathInfo(); + return get(filename, cmdOut); + } + + /* + * @param filename A fuse-mounted filename + * @param xattr A gluster pathinfo extended attribute + * @return A GlusterFSPathInfo + */ + public static GlusterFSPathInfo get(String filename, String xattr) { + GlusterFSPathInfo pathInfo = INVALID; + if (!pathInfoCaches.containsKey(filename)) { + try { + pathInfo = parse(xattr); + } catch (Exception e) { + log.error("Invalid pathinfo: " + e); + } + pathInfoCaches.putIfAbsent(filename,pathInfo); + } + pathInfo = pathInfoCaches.get(filename); + return pathInfo; + } + + /* + * Clear cache + */ + public static void clear() { + pathInfoCaches.clear(); + } + + public String getPathInfo() { + return this.pathInfo; + } + + public long getStripeSize() { + return this.stripeSize; + } + + public List> getLocations() { + return this.locations; + } + + public BlockLocation[] getBlockLocations(long start, long len) { + long stripeSize = this.stripeSize; + List> locations = this.locations; + if (locations == null) + return null; + int numStripes = locations.size(); + int blockStart = (int) (start/stripeSize); + int blockEnd = (int) ((start+len-1)/stripeSize); + int numBlocks = blockEnd-blockStart+1; + BlockLocation[] blkLocations = new BlockLocation[numBlocks]; + for (int i=0; i < numBlocks; i++) { + long startOffset = (blockStart+i)*stripeSize; + long blockLen = Math.min(stripeSize,start+len-startOffset); + int stripeIndex = ((int) (startOffset / stripeSize)) % numStripes; + String[] hosts = locations.get(stripeIndex).toArray(new String[0]); + String[] names = hosts; + blkLocations[i] = new BlockLocation(names,hosts,startOffset,blockLen); + log.debug("Locate block " + blkLocations[i]); + } + return blkLocations; + } + +} From 3d5b30691f7a36b43d6f7955aea5245dc7e5a24c Mon Sep 17 00:00:00 2001 From: Ekasit Kijsipongse Date: Tue, 7 Jul 2015 10:50:36 +0700 Subject: [PATCH 2/2] Update and change pathinfo parsing code --- .../hadoop/fs/glusterfs/GlusterFSXattr.java | 47 +++++----- .../hadoop/fs/glusterfs/GlusterVolume.java | 42 +++++++-- .../hadoop/fs/test/TestBlockLocation.java | 86 ++++++++++--------- 3 files changed, 102 insertions(+), 73 deletions(-) diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSXattr.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSXattr.java index 97ee6675..55e5f514 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSXattr.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSXattr.java @@ -28,8 +28,13 @@ import org.apache.hadoop.fs.BlockLocation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class GlusterFSXattr{ + static final Logger log = LoggerFactory.getLogger(GlusterFSXattr.class); + private String getFattrCmdBase = null; private String filename = null; private String xattrValue = null; @@ -56,43 +61,33 @@ public static String shellToString(String shellCommand) throws IOException{ brInput=new BufferedReader(new InputStreamReader(p.getInputStream())); String value=""; - while ((s=brInput.readLine())!=null) + while ((s=brInput.readLine())!=null) { value+=s; - + } return value; } /* Caches the xattr value. Must call reset() to re-query */ public String execGetFattr() throws IOException{ if(xattrValue==null){ + log.debug("exec " + this.getFattrCmdBase + " " + filename); xattrValue=shellToString(this.getFattrCmdBase + " " + filename); } + log.debug(xattrValue); return xattrValue; } - public BlockLocation[] getPathInfo(long start, long len) { - String xattr = null; - Pattern blockReg = Pattern.compile(""); - try { - xattr = execGetFattr(); - } catch (IOException e) { - // problem executing getfattr command, fail gracefully. - } - - Matcher matcher = blockReg.matcher(xattr); - ArrayList list = new ArrayList(); - while(matcher.find()){ - list.add(matcher.group(1)); - } - - /* no pathinfo found*/ - if(list.size() == 0) - return null; - - String hosts[] = list.toArray(new String[list.size()]); - return new BlockLocation[]{ new BlockLocation(null, hosts, start, len) }; - - } + public String getPathInfo() { + String xattr = null; + try { + xattr = execGetFattr(); + } catch (IOException e) { + log.error(e.toString()); + // problem executing getfattr command, fail gracefully. + xattr = ""; + } + return xattr; + } -} \ No newline at end of file +} diff --git a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java index d271b980..32f4e726 100644 --- a/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java +++ b/src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java @@ -29,6 +29,7 @@ import java.util.Arrays; import java.util.Enumeration; import java.util.Hashtable; +import java.util.List; import java.util.Vector; import org.apache.hadoop.conf.Configuration; @@ -63,7 +64,7 @@ public class GlusterVolume extends RawLocalFileSystem{ protected int tsPrecisionChop; protected static GlusterFSXattr attr = null; - + public GlusterVolume(){} public GlusterVolume(Configuration conf){ @@ -328,7 +329,7 @@ public FileStatus[] listStatus(Path f) throws IOException { } if (localf.isFile()) { return new FileStatus[] { - new GlusterFileStatus(localf, getDefaultBlockSize(), this) }; + new GlusterFileStatus(localf, getBlockSize(f), this) }; } if(localf.isDirectory() && !localf.canRead()){ @@ -379,14 +380,20 @@ public FileStatus getFileStatus(Path f) throws IOException { } if (path.exists()) { - return new GlusterFileStatus(pathToFile(f), getDefaultBlockSize(), this); + return new GlusterFileStatus(path, getBlockSize(f), this); } else { throw new FileNotFoundException( "File " + f + " does not exist."); } } public long getBlockSize(Path path) throws IOException{ - return getLength(path); + File f = pathToFile(path); + GlusterFSPathInfo pathInfo = GlusterFSPathInfo.get(f.getPath()); + long stripeSize = Long.MAX_VALUE; + if (pathInfo != null) { + stripeSize = pathInfo.getStripeSize(); + } + return Math.min(stripeSize,getDefaultBlockSize()); // Should we use f.length() ? } public void setOwner(Path p, String username, String groupname) @@ -400,14 +407,33 @@ public void setPermission(Path p, FsPermission permission) super.setPermission(p,permission); } - public BlockLocation[] getFileBlockLocations(FileStatus file,long start,long len) throws IOException{ + public BlockLocation[] getFileBlockLocations(FileStatus file,long start,long len) throws IOException { + if (file == null) { + return null; + } + + if ((start < 0) || (len <= 0)) { + log.error("Invalid start or len " + start + "," + len); + return null; + } + + if (file.getLen() <= start) { + return new BlockLocation[0]; + } + File f=pathToFile(file.getPath()); - BlockLocation[] result = new GlusterFSXattr(f.getPath()).getPathInfo(start, len); - if(result==null){ + GlusterFSPathInfo pathInfo = GlusterFSPathInfo.get(f.getPath()); + + if (pathInfo == null) { + return null; + } + + BlockLocation[] blkLocations = pathInfo.getBlockLocations(start,len); + if(blkLocations == null){ log.info("GLUSTERFS: Problem getting host/block location for file "+f.getPath()); } - return result; + return blkLocations; } public String toString(){ diff --git a/src/test/java/org/apache/hadoop/fs/test/TestBlockLocation.java b/src/test/java/org/apache/hadoop/fs/test/TestBlockLocation.java index 901f7d2f..506dae2f 100644 --- a/src/test/java/org/apache/hadoop/fs/test/TestBlockLocation.java +++ b/src/test/java/org/apache/hadoop/fs/test/TestBlockLocation.java @@ -6,8 +6,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.glusterfs.GlusterFSPathInfo; import org.apache.hadoop.fs.glusterfs.GlusterFSXattr; import org.apache.hadoop.fs.test.connector.HcfsTestConnectorFactory; import org.apache.hadoop.fs.test.connector.HcfsTestConnectorInterface; @@ -16,43 +18,36 @@ public class TestBlockLocation { - /* - * Test subclass to simulate / inject xattr values instead of requiring - * filesystem. - * - */ - class TestBlockLocationHarnace extends GlusterFSXattr { - String xattrValue = null; + FileStatus fs = new FileStatus(100000,false,0,0,0,new Path("/a")); - TestBlockLocationHarnace(String xattrValue) { - super(null); - this.xattrValue = xattrValue; - } + @Test + public void testDistributedVolume() throws IOException { + String xattr = "trusted.glusterfs.pathinfo=\"( )\""; - public String execGetFattr() throws IOException { - return this.xattrValue; - } + GlusterFSPathInfo.clear(); + GlusterFSPathInfo gpi = GlusterFSPathInfo.get(fs.getPath().toString(),xattr); + BlockLocation[] blocks = gpi.getBlockLocations(10000, 20000); + assertTrue(blocks.length == 1); + assertTrue(blocks[0].getHosts()[0].equals("vm-2")); } @Test - public void testReplicateVolume() throws IOException { - String xattr = "trusted.glusterfs.pathinfo=\"( ( )\""; - xattr = xattr.substring(28, xattr.length() - 1); - TestBlockLocationHarnace tbl = new TestBlockLocationHarnace(xattr); + public void testStripeVolume() throws IOException { + String xattr = "trusted.glusterfs.pathinfo=\"( ( ))\""; - int start = 0; - int len = 0; - BlockLocation[] blocks = tbl.getPathInfo(start, len); + GlusterFSPathInfo.clear(); + GlusterFSPathInfo gpi = GlusterFSPathInfo.get(fs.getPath().toString(),xattr); + BlockLocation[] blocks = gpi.getBlockLocations(0, 10000); assertTrue(blocks.length == 1); assertTrue(blocks[0].getHosts()[0].equals("vm-1")); } @Test - public void testReplicateVolume2() throws IOException { + public void testReplicateVolume() throws IOException { String xattr = "trusted.glusterfs.pathinfo=\"( ( ))\""; - xattr = xattr.substring(28, xattr.length() - 1); - TestBlockLocationHarnace tbl = new TestBlockLocationHarnace(xattr); - BlockLocation[] blocks = tbl.getPathInfo(0, 10000); + GlusterFSPathInfo.clear(); + GlusterFSPathInfo gpi = GlusterFSPathInfo.get(fs.getPath().toString(),xattr); + BlockLocation[] blocks = gpi.getBlockLocations(0, 10000); assertTrue(blocks.length == 1); String hosts[] = blocks[0].getHosts(); @@ -65,10 +60,11 @@ public void testReplicateVolume2() throws IOException { * Make sure the xattr code fails gracefull when the input is garbage. */ @Test - public void testFailureVolumeGarbage() { + public void testFailureVolumeGarbage() throws IOException { String xattr = "dsfsffsfdsf"; - TestBlockLocationHarnace tbl = new TestBlockLocationHarnace(xattr); - BlockLocation[] blocks = tbl.getPathInfo(0, 10000); + GlusterFSPathInfo.clear(); + GlusterFSPathInfo gpi = GlusterFSPathInfo.get(fs.getPath().toString(),xattr); + BlockLocation[] blocks = gpi.getBlockLocations(0, 10000); assertNull(blocks); } @@ -77,26 +73,38 @@ public void testFailureVolumeGarbage() { */ @Test - public void testFailureVolumeEmpty() { + public void testFailureVolumeEmpty() throws IOException { String xattr = ""; - TestBlockLocationHarnace tbl = new TestBlockLocationHarnace(xattr); - BlockLocation[] blocks = tbl.getPathInfo(0, 10000); + GlusterFSPathInfo.clear(); + GlusterFSPathInfo gpi = GlusterFSPathInfo.get(fs.getPath().toString(),xattr); + BlockLocation[] blocks = gpi.getBlockLocations(0, 10000); assertNull(blocks); } /* * - * stripe not yet supported.trusted.pathinfo is incorrect in gluster: + * Require a patch to fix trusted.glusterfs.pathinfo in gluster stripe volume. * https://bugzilla.redhat.com/show_bug.cgi?id=1200914 */ - - public void testStripeVolume() { - String xattr = "trusted.glusterfs.pathinfo=\"( ( ( )( )( ))\""; + @Test + public void testStripeReplicateVolume() throws IOException { + String xattr = "trusted.glusterfs.pathinfo=\"( ( ( )( )( )))\""; ; - xattr = xattr.substring(28, xattr.length() - 1); - TestBlockLocationHarnace tbl = new TestBlockLocationHarnace(xattr); - BlockLocation[] blocks = tbl.getPathInfo(0, 10000); - + GlusterFSPathInfo.clear(); + GlusterFSPathInfo gpi = GlusterFSPathInfo.get(fs.getPath().toString(),xattr); + BlockLocation[] blocks = gpi.getBlockLocations(200000, 500000); + assertTrue(blocks.length == 5); + assertTrue(blocks[0].getHosts()[0].equals("vm-3")); + assertTrue(blocks[0].getHosts()[1].equals("vm-4")); + assertTrue(blocks[1].getHosts()[0].equals("vm-5")); + assertTrue(blocks[1].getHosts()[1].equals("vm-6")); + assertTrue(blocks[2].getHosts()[0].equals("vm-1")); + assertTrue(blocks[2].getHosts()[1].equals("vm-2")); + assertTrue(blocks[3].getHosts()[0].equals("vm-3")); + assertTrue(blocks[3].getHosts()[1].equals("vm-4")); + assertTrue(blocks[4].getHosts()[0].equals("vm-5")); + assertTrue(blocks[4].getHosts()[1].equals("vm-6")); + } }