Merge branch 'master-ripme' into issues/1402_confirm_clear

pokifan · Aug 23, 2019 · 6a9f44f · 6a9f44f
2 parents b47c376 + dcc1d2f
commit 6a9f44f
Show file tree

Hide file tree

Showing 60 changed files with 241 additions and 114 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,6 +1,12 @@
 language: java
-jdk:
-  - oraclejdk8
-  - openjdk8
+
+matrix:
+  include:
+  - jdk: openjdk9
+    before_install:
+    - rm "${JAVA_HOME}/lib/security/cacerts"
+    - ln -s /etc/ssl/certs/java/cacerts "${JAVA_HOME}/lib/security/cacerts"
+  - jdk: openjdk8
+
 after_success:
   - mvn clean test jacoco:report coveralls:report
diff --git a/pom.xml b/pom.xml
@@ -3,7 +3,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.7.86</version>
+  <version>1.7.87</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>

diff --git a/ripme.json b/ripme.json
@@ -1,7 +1,7 @@
 {
-    "latestVersion": "1.7.86",
-    "currentHash": "d78f7dfaa8ef55575a8485cdf93bbd09e3ea9a3bd224e84e970bcdd0a51d5305",
+    "currentHash": "52dfb707d6247f44949c0d97b19c7815dc848b26837b98ae561c0dea20993a12",
     "changeList": [
+        "1.7.87: Added ripper for allporncomic.com; Fixed Xhamster ripper; Added support xhamster2.com and xhamster.desi; Fixes for gfycat thumbs urls",
         "1.7.86: Added Meituri Ripper; fixed -u flag; Fixed pornhub ripper; Xhamster ripper can now queue users videos",
         "1.7.85: Fixed instagram ripper; Flickr ripper now downloads largest image",
         "1.7.84: Fixed instagram ripper; xhamster ripper now accepts urls with page numbers; Fixed Deviantart Ripper",
@@ -258,5 +258,6 @@
         "1.0.4: Fixed spaces-in-directory bug",
         "1.0.3: Added VK.com ripper",
         "1.0.1: Added auto-update functionality"
-    ]
+    ],
+    "latestVersion": "1.7.87"
 }
diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java
@@ -29,7 +29,7 @@ public class GfycatRipper extends AbstractHTMLRipper {
 
 
     public GfycatRipper(URL url) throws IOException {
-        super(url);
+        super(new URL(url.toExternalForm().split("-")[0].replace("thumbs.", "")));
     }
 
     @Override
@@ -76,23 +76,24 @@ public void downloadURL(URL url, int index) {
 
     @Override
     public String getGID(URL url) throws MalformedURLException {
-        Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/@?([a-zA-Z0-9]+).*$");
+        Pattern p = Pattern.compile("^https?://(thumbs\\.|[wm\\.]*)gfycat\\.com/@?([a-zA-Z0-9]+).*$");
         Matcher m = p.matcher(url.toExternalForm());
-        if (m.matches()) {
-            return m.group(1);
-        }
-
+
+        if (m.matches())
+            return m.group(2);
+        
         throw new MalformedURLException(
-                "Expected gfycat.com format:"
-                        + "gfycat.com/id"
+                "Expected gfycat.com format: "
+                        + "gfycat.com/id or "
+                        + "thumbs.gfycat.com/id.gif"
                         + " Got: " + url);
     }
 
     private String stripHTMLTags(String t) {
         t = t.replaceAll("<html>\n" +
                 " <head></head>\n" +
                 " <body>", "");
-        t.replaceAll("</body>\n" +
+        t = t.replaceAll("</body>\n" +
                 "</html>", "");
         t = t.replaceAll("\n", "");
         t = t.replaceAll("=\"\"", "");

diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
@@ -17,7 +17,7 @@
 
 // WARNING
 // This ripper changes all requests to use the MOBILE version of the site
-// If you're chaning anything be sure to use the mobile sites html/css or you\re just wasting your time!
+// If you're changing anything be sure to use the mobile sites html/css or you're just wasting your time!
 // WARNING
 
 public class XhamsterRipper extends AbstractHTMLRipper {
@@ -44,31 +44,30 @@ public String getDomain() {
 
     @Override
     public URL sanitizeURL(URL url) throws MalformedURLException {
-        String URLToReturn = url.toExternalForm();
-        URLToReturn = URLToReturn.replaceAll("xhamster.one", "xhamster.com");
-        URLToReturn = URLToReturn.replaceAll("m.xhamster.com", "xhamster.com");
-        URLToReturn = URLToReturn.replaceAll("\\w\\w.xhamster.com", "xhamster.com");
-        if (!isVideoUrl(url)) {
-            URLToReturn = URLToReturn.replaceAll("xhamster.com", "m.xhamster.com");
+        if (isVideoUrl(url)) {
+            return url;
         }
+        String URLToReturn = url.toExternalForm();
+        URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster\\.", "https://m.xhamster.");
+        URLToReturn = URLToReturn.replaceAll("https?://xhamster2\\.", "https://m.xhamster2.");
         URL san_url = new URL(URLToReturn);
         LOGGER.info("sanitized URL is " + san_url.toExternalForm());
         return san_url;
     }
 
     @Override
     public String getGID(URL url) throws MalformedURLException {
-        Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster\\.com/photos/gallery/.*?(\\d+)$");
+        Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.com/photos/gallery/.*?(\\d+)$");
         Matcher m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             return m.group(1);
         }
-        p = Pattern.compile("^https?://[\\w\\w.]*xhamster\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?");
+        p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?");
         m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             return "user_" + m.group(1);
         }
-        p = Pattern.compile("^https?://.*xhamster\\.com/(movies|videos)/(.*)$");
+        p = Pattern.compile("^https?://.*xhamster2?\\.com/(movies|videos)/(.*)$");
         m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             return m.group(2);
@@ -114,17 +113,17 @@ public Document getFirstPage() throws IOException {
 
     @Override
     public boolean canRip(URL url) {
-        Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster\\.(com|one)/photos/gallery/.*?(\\d+)$");
+        Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster2?\\.(com|one|desi)/photos/gallery/.*?(\\d+)$");
         Matcher m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             return true;
         }
-        p = Pattern.compile("^https?://[\\w\\w.]*xhamster\\.(com|one)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?");
+        p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?");
         m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             return true;
         }
-        p = Pattern.compile("^https?://.*xhamster\\.(com|one)/(movies|videos)/.*$");
+        p = Pattern.compile("^https?://.*xhamster2?\\.(com|one|desi)/(movies|videos)/.*$");
         m = p.matcher(url.toExternalForm());
         if (m.matches()) {
             return true;
@@ -133,7 +132,7 @@ public boolean canRip(URL url) {
     }
 
     private boolean isVideoUrl(URL url) {
-        Pattern p = Pattern.compile("^https?://.*xhamster\\.com/(movies|videos)/.*$");
+        Pattern p = Pattern.compile("^https?://.*xhamster2?\\.(com|one|desi)/(movies|videos)/.*$");
         Matcher m = p.matcher(url.toExternalForm());
         return m.matches();
     }
@@ -155,9 +154,17 @@ public List<String> getURLsFromPage(Document doc) {
         List<String> result = new ArrayList<>();
         if (!isVideoUrl(url)) {
           for (Element page : doc.select("div.items > div.item-container > a.item")) {
+              // Make sure we don't waste time running the loop if the ripper has been stopped
+              if (isStopped()) {
+                  break;
+              }
               String pageWithImageUrl = page.attr("href");
               try {
-                  String image = Http.url(new URL(pageWithImageUrl)).get().select("div.picture_container > a > img").attr("src");
+                  // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to
+                  // the page chamster.com but displays the mobile site from m.xhamster.com
+                  pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster\\.", "://m.xhamster.");
+                  pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster2\\.", "://m.xhamster.");
+                  String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src");
                   downloadFile(image);
               } catch (IOException e) {
                   LOGGER.error("Was unable to load page " + pageWithImageUrl);

diff --git a/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java b/src/main/java/com/rarchives/ripme/ui/UpdateUtils.java
@@ -23,7 +23,7 @@
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.7.86";
+    private static final String DEFAULT_VERSION = "1.7.87";
     private static final String REPO_NAME = "ripmeapp/ripme";
     private static final String updateJsonURL = "https://raw.githubusercontent.com/" + REPO_NAME + "/master/ripme.json";
     private static String mainFileName;
@@ -264,12 +264,14 @@ private static void downloadJarAndLaunch(String updateJarURL, Boolean shouldLaun
             // Windows
             final String batchFile = "update_ripme.bat";
             final String batchPath = new File(batchFile).getAbsolutePath();
-            String script = "@echo off\r\n" + "timeout 1\r\n" + "copy " + updateFileName + " " + mainFileName + "\r\n"
-                    + "del " + updateFileName + "\r\n";
-            if (shouldLaunch) {
-                script += mainFileName + "\r\n";
-            }
-            script += "del " + batchPath + "\r\n";
+            String script = "@echo off\r\n" + "timeout 1\r\n" 
+                    + "copy \"" + updateFileName + "\" \"" + mainFileName + "\"\r\n"
+                    + "del \"" + updateFileName + "\"\r\n";
+
+            if (shouldLaunch) 
+                script += "\"" + mainFileName + "\"\r\n";
+            script += "del \"" + batchPath + "\"\r\n";
+
             final String[] batchExec = new String[] { batchPath };
             // Create updater script
             try (BufferedWriter bw = new BufferedWriter(new FileWriter(batchFile))) {

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ChanRipperTest.java
@@ -11,9 +11,10 @@
 import com.rarchives.ripme.ripper.rippers.ripperhelpers.ChanSite;
 import com.rarchives.ripme.utils.Http;
 import org.jsoup.nodes.Document;
+import org.junit.jupiter.api.Test;
 
 public class ChanRipperTest extends RippersTest {
-
+    @Test
     public void testChanURLFailures() throws IOException {
         List<URL> failURLs = new ArrayList<>();
         // URLs that should not work
@@ -26,7 +27,7 @@ public void testChanURLFailures() throws IOException {
             }
         }
     }
-
+    @Test
     public void testChanURLPasses() throws IOException {
         List<URL> passURLs = new ArrayList<>();
         // URLs that should work
@@ -43,7 +44,7 @@ public void testChanURLPasses() throws IOException {
             deleteDir(ripper.getWorkingDir());
         }
     }
-
+    @Test
     public void testChanStringParsing() throws IOException {
         List<String> site1 = Arrays.asList("site1.com");
         List<String> site1Cdns = Arrays.asList("cnd1.site1.com", "cdn2.site2.biz");
@@ -59,7 +60,7 @@ public void testChanStringParsing() throws IOException {
         assertEquals(chansFromConfig.get(1).getDomains(), site2);
         assertEquals(chansFromConfig.get(1).getCdns(), site2Cdns);
     }
-
+    @Test
     public void testChanRipper() throws IOException {
         List<URL> contentURLs = new ArrayList<>();
         contentURLs.add(new URL(getRandomThreadDesuarchive()));

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/CheveretoRipperTest.java
@@ -4,13 +4,15 @@
 import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.CheveretoRipper;
+import org.junit.jupiter.api.Test;
 
 public class CheveretoRipperTest extends RippersTest {
+    @Test
     public void testTagFox() throws IOException {
         CheveretoRipper ripper = new CheveretoRipper(new URL("http://tag-fox.com/album/Thjb"));
         testRipper(ripper);
     }
-
+    @Test
     public void testSubdirAlbum() throws IOException {
         CheveretoRipper ripper = new CheveretoRipper(new URL("https://kenzato.uk/booru/album/TnEc"));
         testRipper(ripper);

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ComicextraRipperTest.java
@@ -3,15 +3,16 @@
 import java.io.IOException;
 import java.net.URL;
 import com.rarchives.ripme.ripper.rippers.ComicextraRipper;
+import org.junit.jupiter.api.Test;
 
 public class ComicextraRipperTest extends RippersTest {
-
+    @Test
     public void testComicUrl() throws IOException {
         URL url = new URL("https://www.comicextra.com/comic/karma-police");
         ComicextraRipper ripper = new ComicextraRipper(url);
         testRipper(ripper);
     }
-
+    @Test
     public void testChapterUrl() throws IOException {
         URL url = new URL("https://www.comicextra.com/v-for-vendetta/chapter-1");
         ComicextraRipper ripper = new ComicextraRipper(url);

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DribbbleRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/DribbbleRipperTest.java
@@ -4,8 +4,10 @@
 import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.DribbbleRipper;
+import org.junit.jupiter.api.Test;
 
 public class DribbbleRipperTest extends RippersTest {
+    @Test
     public void testDribbbleRip() throws IOException {
         DribbbleRipper ripper = new DribbbleRipper(new URL("https://dribbble.com/typogriff"));
         testRipper(ripper);

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java
@@ -4,18 +4,19 @@
 import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.E621Ripper;
+import org.junit.jupiter.api.Test;
 
 public class E621RipperTest extends RippersTest {
     public void testRip() throws IOException {
         E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/beach"));
         testRipper(ripper);
     }
-
+    @Test
     public void testFlashOrWebm() throws IOException {
         E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/gif"));
         testRipper(ripper);
     }
-
+    @Test
     public void testGetNextPage() throws IOException {
         E621Ripper nextPageRipper = new E621Ripper(new URL("https://e621.net/post/index/1/cosmicminerals"));
         try {

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EhentaiRipperTest.java
@@ -6,14 +6,17 @@
 
 import com.rarchives.ripme.ripper.rippers.EHentaiRipper;
 import com.rarchives.ripme.utils.RipUtils;
+import org.junit.jupiter.api.Test;
 
 public class EhentaiRipperTest extends RippersTest {
+    @Test
     public void testEHentaiAlbum() throws IOException {
         EHentaiRipper ripper = new EHentaiRipper(new URL("https://e-hentai.org/g/1144492/e823bdf9a5/"));
         testRipper(ripper);
     }
 
     // Test the tag black listing
+    @Test
     public void testTagBlackList()  throws IOException {
         URL url = new URL("https://e-hentai.org/g/1228503/1a2f455f96/");
         EHentaiRipper ripper = new EHentaiRipper(url);

diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/EightmusesRipperTest.java
@@ -4,8 +4,10 @@
 import java.net.URL;
 
 import com.rarchives.ripme.ripper.rippers.EightmusesRipper;
+import org.junit.jupiter.api.Test;
 
 public class EightmusesRipperTest extends RippersTest {
+    @Test
     public void testEightmusesAlbum() throws IOException {
         // A simple image album
         EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore"));
@@ -17,12 +19,12 @@ public void testEightmusesAlbum() throws IOException {
         ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Blacknwhitecomics_com-Comix/BlacknWhiteComics/The-Mayor"));
         testRipper(ripper);
     }
-
+    @Test
     public void testGID() throws IOException {
         EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore"));
         assertEquals("Affect3D-Comics", ripper.getGID(new URL("https://www.8muses.com/comics/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore")));
     }
-
+    @Test
     public void testGetSubdir() throws IOException {
         EightmusesRipper ripper = new EightmusesRipper(new URL("https://www.8muses.com/comix/album/Affect3D-Comics/TheDude3DX/Lust-Unleashed-The-Urge-To-Explore"));
         assertEquals("After-Party-Issue-1", ripper.getSubdir("After Party - Issue 1"));