add filter to update value or do something

Author: fivesmallq

src/main/java/im/nll/data/extractor/Extractors.java

@@ -24,6 +24,7 @@ public class Extractors {
     private String html;
     private List<String> htmlList;
     private Map<String, List<Extractor>> extractorsMap = Maps.newLinkedHashMap();
+    private Map<String, List<Filter>> filtersMap = Maps.newLinkedHashMap();
     private String prevField;
 
     public Extractors(String html) {
@@ -55,6 +56,14 @@ public Extractors with(Extractor extractor) {
         return this;
     }
 
+    public Extractors filter(Filter filter) {
+        Validate.notNull(prevField, "must call extract method first!");
+        List<Filter> filters = filtersMap.getOrDefault(prevField, Lists.newLinkedList());
+        filters.add(filter);
+        filtersMap.put(prevField, filters);
+        return this;
+    }
+
 
     /**
      * split html use listable extractor
@@ -74,6 +83,10 @@ public String asString() {
         for (Extractor extractor : extractors) {
             result = extractor.extract(result);
         }
+        List<Filter> filters = filtersMap.getOrDefault(DEFAULT_FIELD, Lists.newLinkedList());
+        for (Filter filter : filters) {
+            result = filter.process(result);
+        }
         return result;
     }
 
@@ -86,6 +99,10 @@ public Map<String, String> asMap() {
             for (Extractor extractor : extractors) {
                 result = extractor.extract(result);
             }
+            List<Filter> filters = filtersMap.getOrDefault(name, Lists.newLinkedList());
+            for (Filter filter : filters) {
+                result = filter.process(result);
+            }
             try {
                 map.put(name, result);
             } catch (Exception e) {
@@ -107,6 +124,10 @@ public List<Map<String, String>> asMapList() {
                 for (Extractor extractor : extractors) {
                     result = extractor.extract(result);
                 }
+                List<Filter> filters = filtersMap.getOrDefault(name, Lists.newLinkedList());
+                for (Filter filter : filters) {
+                    result = filter.process(result);
+                }
                 map.put(name, result);
             }
             mapList.add(map);
@@ -124,6 +145,10 @@ public <T> T asBean(Class<T> clazz) {
             for (Extractor extractor : extractors) {
                 result = extractor.extract(result);
             }
+            List<Filter> filters = filtersMap.getOrDefault(name, Lists.newLinkedList());
+            for (Filter filter : filters) {
+                result = filter.process(result);
+            }
             try {
                 Reflect.on(entity).set(name, result);
             } catch (Exception e) {
@@ -150,6 +175,10 @@ public <T> List<T> asBeanList(Class<T> clazz) {
                 for (Extractor extractor : extractors) {
                     result = extractor.extract(result);
                 }
+                List<Filter> filters = filtersMap.getOrDefault(name, Lists.newLinkedList());
+                for (Filter filter : filters) {
+                    result = filter.process(result);
+                }
                 try {
                     Reflect.on(entity).set(name, result);
                 } catch (Exception e) {

src/main/java/im/nll/data/extractor/Filter.java

@@ -0,0 +1,12 @@
+package im.nll.data.extractor;
+
+/**
+ * Value Filter
+ *
+ * @author <a href="mailto:[email protected]">fivesmallq</a>
+ * @version Revision: 1.0
+ * @date 16/1/11 下午6:17
+ */
+public interface Filter {
+    String process(String value);
+}

src/test/java/im/nll/data/extractor/ExtractorsTest.java

@@ -45,6 +45,17 @@ public void testGet() throws Exception {
         Assert.assertEquals("Talk is cheap. Show me the code.", description);
     }
 
+    @Test
+    public void testGetWithFilter() throws Exception {
+        String title = Extractors.on(baseHtml).extract(selector("a.title")).asString();
+        String followers = Extractors.on(baseHtml).extract(selector("div.followers")).with(regex("\\d+")).asString();
+        //use filter to process value
+        String description = Extractors.on(baseHtml).extract(selector("div.description")).filter(value -> value.toLowerCase()).asString();
+        Assert.assertEquals("fivesmallq", title);
+        Assert.assertEquals("29671", followers);
+        Assert.assertEquals("talk is cheap. show me the code.", description);
+    }
+
     @Test
     public void testToMap() throws Exception {
         Map<String, String> dataMap = Extractors.on(baseHtml)
@@ -114,6 +125,21 @@ public void testToBeanListByXPath() throws Exception {
         Assert.assertEquals(second.getUrl(), "https://www.ruby-lang.org");
     }
 
+    @Test
+    public void testToBeanListFilter() throws Exception {
+        List<Language> languages = Extractors.on(listHtml).split(xpath("//tr[@class='item']"))
+                .extract("type", xpath("//td[1]/text()")).filter(value -> "type:" + value)
+                .extract("name", xpath("//td[2]/text()")).filter(value -> "name:" + value)
+                .extract("url", xpath("//td[3]/text()")).filter(value -> "url:" + value)
+                .asBeanList(Language.class);
+        Assert.assertNotNull(languages);
+        Language second = languages.get(1);
+        Assert.assertEquals(languages.size(), 3);
+        Assert.assertEquals(second.getType(), "type:dynamic");
+        Assert.assertEquals(second.getName(), "name:Ruby");
+        Assert.assertEquals(second.getUrl(), "url:https://www.ruby-lang.org");
+    }
+
     @Test
     public void testToBeanListByJson() throws Exception {
         List<Book> books = Extractors.on(jsonString).split(json("$..book.*"))