Merge remote-tracking branch 'upstream/dev' into dev

jerrytaotz · Apr 11, 2020 · 6543a74 · 6543a74
2 parents 0d3fba6 + ed4ec48
commit 6543a74
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 8 deletions.
diff --git a/.../main/java/org/spiderflow/core/executor/function/extension/ElementsFunctionExtension.java b/.../main/java/org/spiderflow/core/executor/function/extension/ElementsFunctionExtension.java
@@ -1,16 +1,16 @@
 package org.spiderflow.core.executor.function.extension;
 
-import java.util.List;
-
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 import org.spiderflow.annotation.Comment;
 import org.spiderflow.annotation.Example;
-import org.spiderflow.annotation.Return;
 import org.spiderflow.core.utils.ExtractUtils;
 import org.spiderflow.executor.FunctionExtension;
 import org.springframework.stereotype.Component;
 
+import java.util.ArrayList;
+import java.util.List;
+
 @Component
 public class ElementsFunctionExtension implements FunctionExtension{
 
@@ -21,14 +21,12 @@ public Class<?> support() {
 
 	@Comment("根据xpath提取内容")
 	@Example("${elementsVar.xpath('//title/text()')}")
-	@Return({Element.class,String.class})
 	public static String xpath(Elements elements,String xpath){
 		return ExtractUtils.getValueByXPath(elements, xpath);
 	}
 
 	@Comment("根据xpath提取内容")
 	@Example("${elementsVar.xpaths('//h2/text()')}")
-	@Return({Element.class,String.class})
 	public static List<String> xpaths(Elements elements,String xpath){
 		return ExtractUtils.getValuesByXPath(elements, xpath);
 	}
@@ -78,6 +76,76 @@ public static Element selector(Elements elements,String selector){
 		}
 		return null;
 	}
+
+	@Comment("返回所有attr")
+	@Example("${elementsVar.attrs('href')}")
+	public static List<String> attrs(Elements elements,String key){
+		List<String> list = new ArrayList<>(elements.size());
+		for (Element element : elements) {
+			list.add(element.attr(key));
+		}
+		return list;
+	}
+
+	@Comment("返回所有value")
+	@Example("${elementsVar.vals()}")
+	public static List<String> vals(Elements elements){
+		List<String> list = new ArrayList<>(elements.size());
+		for (Element element : elements) {
+			list.add(element.val());
+		}
+		return list;
+	}
+
+	@Comment("返回所有text")
+	@Example("${elementsVar.texts()}")
+	public static List<String> texts(Elements elements){
+		List<String> list = new ArrayList<>(elements.size());
+		for (Element element : elements) {
+			list.add(element.text());
+		}
+		return list;
+	}
+
+	@Comment("返回所有html")
+	@Example("${elementsVar.htmls()}")
+	public static List<String> htmls(Elements elements){
+		List<String> list = new ArrayList<>(elements.size());
+		for (Element element : elements) {
+			list.add(element.html());
+		}
+		return list;
+	}
+
+	@Comment("返回所有outerHtml")
+	@Example("${elementsVar.outerHtmls()}")
+	public static List<String> outerHtmls(Elements elements){
+		List<String> list = new ArrayList<>(elements.size());
+		for (Element element : elements) {
+			list.add(element.outerHtml());
+		}
+		return list;
+	}
+
+	@Comment("返回所有ownTexts")
+	@Example("${elementsVar.ownTexts()}")
+	public static List<String> ownTexts(Elements elements){
+		List<String> list = new ArrayList<>(elements.size());
+		for (Element element : elements) {
+			list.add(element.ownText());
+		}
+		return list;
+	}
+
+	@Comment("返回所有wholeText")
+	@Example("${elementsVar.wholeTexts()}")
+	public static List<String> wholeTexts(Elements elements){
+		List<String> list = new ArrayList<>(elements.size());
+		for (Element element : elements) {
+			list.add(element.wholeText());
+		}
+		return list;
+	}
 
 	@Comment("根据css选择器提取内容")
 	@Example("${elementsVar.selectors('div > a')}")

diff --git a/.../main/java/org/spiderflow/core/executor/function/extension/ResponseFunctionExtension.java b/.../main/java/org/spiderflow/core/executor/function/extension/ResponseFunctionExtension.java
@@ -27,7 +27,7 @@ public Class<?> support() {
     @Comment("将请求结果转为Element对象")
     @Example("${resp.element()}")
     public static Element element(SpiderResponse response) {
-        return Jsoup.parse(response.getHtml());
+        return Jsoup.parse(response.getHtml(),response.getUrl());
     }
 
     @Comment("根据xpath在请求结果中查找")

diff --git a/...ore/src/main/java/org/spiderflow/core/executor/function/extension/SqlRowSetExtension.java b/...ore/src/main/java/org/spiderflow/core/executor/function/extension/SqlRowSetExtension.java
@@ -21,8 +21,7 @@ public Class<?> support() {
     @Example("${rs.nextToMap()}")
     public static Map<String, Object> nextToMap(SqlRowSet sqlRowSet) {
         try {
-            boolean next = sqlRowSet.next();
-            if (!next) {
+            if (!sqlRowSet.next()) {
                 return null;
             }
             String[] columnNames = sqlRowSet.getMetaData().getColumnNames();