1

I have a simple Java code with HTMLUnit library, which should print a title of the site.

public class Main {
    public static void main(String[] args) {
        try (final WebClient webClient = new WebClient()) {
            webClient.getOptions().setUseInsecureSSL(true);
            final HtmlPage mainPage = webClient.getPage("https://www.sketchfab.com/");
            System.out.println(mainPage.getTitleText());
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

It works for https://www.google.com/, but doesn't work for https://sketchfab.com/. It produces exception:

======= EXCEPTION START ========
Exception class=[net.sourceforge.htmlunit.corejs.javascript.WrappedException]
com.gargoylesoftware.htmlunit.ScriptException: Wrapped com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException: 404 Not Found for https://player.vimeo.com/video/289701084/fallback?js&referrer=https%3A%2F%2Fsketchfab.com%2F
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine$HtmlUnitContextAction.run(JavaScriptEngine.java:892)
    at net.sourceforge.htmlunit.corejs.javascript.Context.call(Context.java:616)
    at net.sourceforge.htmlunit.corejs.javascript.ContextFactory.call(ContextFactory.java:534)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.execute(JavaScriptEngine.java:772)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.execute(JavaScriptEngine.java:748)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.execute(JavaScriptEngine.java:739)
    at com.gargoylesoftware.htmlunit.html.HtmlPage.executeJavaScript(HtmlPage.java:922)
    at com.gargoylesoftware.htmlunit.html.HtmlScript.executeInlineScriptIfNeeded(HtmlScript.java:316)
    at com.gargoylesoftware.htmlunit.html.HtmlScript.executeScriptIfNeeded(HtmlScript.java:396)
    at com.gargoylesoftware.htmlunit.html.HtmlScript$2.execute(HtmlScript.java:246)
    at com.gargoylesoftware.htmlunit.html.HtmlScript.onAllChildrenAddedToPage(HtmlScript.java:267)
    at com.gargoylesoftware.htmlunit.html.HTMLParser$HtmlUnitDOMBuilder.endElement(HTMLParser.java:802)
    at org.apache.xerces.parsers.AbstractSAXParser.endElement(Unknown Source)
    at com.gargoylesoftware.htmlunit.html.HTMLParser$HtmlUnitDOMBuilder.endElement(HTMLParser.java:758)
    at net.sourceforge.htmlunit.cyberneko.HTMLTagBalancer.callEndElement(HTMLTagBalancer.java:1194)
    at net.sourceforge.htmlunit.cyberneko.HTMLTagBalancer.endElement(HTMLTagBalancer.java:1134)
    at net.sourceforge.htmlunit.cyberneko.filters.DefaultFilter.endElement(DefaultFilter.java:221)
    at net.sourceforge.htmlunit.cyberneko.filters.NamespaceBinder.endElement(NamespaceBinder.java:314)
    at net.sourceforge.htmlunit.cyberneko.HTMLScanner$ContentScanner.scanEndElement(HTMLScanner.java:3179)
    at net.sourceforge.htmlunit.cyberneko.HTMLScanner$ContentScanner.scan(HTMLScanner.java:2132)
    at net.sourceforge.htmlunit.cyberneko.HTMLScanner.scanDocument(HTMLScanner.java:939)
    at net.sourceforge.htmlunit.cyberneko.HTMLConfiguration.parse(HTMLConfiguration.java:452)
    at net.sourceforge.htmlunit.cyberneko.HTMLConfiguration.parse(HTMLConfiguration.java:403)
    at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
    at com.gargoylesoftware.htmlunit.html.HTMLParser$HtmlUnitDOMBuilder.parse(HTMLParser.java:1001)
    at com.gargoylesoftware.htmlunit.html.HTMLParser.parse(HTMLParser.java:250)
    at com.gargoylesoftware.htmlunit.html.HTMLParser.parseHtml(HTMLParser.java:196)
    at com.gargoylesoftware.htmlunit.DefaultPageCreator.createHtmlPage(DefaultPageCreator.java:267)
    at com.gargoylesoftware.htmlunit.DefaultPageCreator.createPage(DefaultPageCreator.java:158)
    at com.gargoylesoftware.htmlunit.WebClient.loadWebResponseInto(WebClient.java:531)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:398)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:315)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement.loadInnerPageIfPossible(BaseFrameElement.java:184)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement.loadInnerPage(BaseFrameElement.java:121)
    at com.gargoylesoftware.htmlunit.html.HtmlPage.loadFrames(HtmlPage.java:1894)
    at com.gargoylesoftware.htmlunit.html.HtmlPage.initialize(HtmlPage.java:251)
    at com.gargoylesoftware.htmlunit.WebClient.loadWebResponseInto(WebClient.java:538)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:398)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:315)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:466)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:448)
    at Main.main(Main.java:19)
Caused by: net.sourceforge.htmlunit.corejs.javascript.WrappedException: Wrapped com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException: 404 Not Found for https://player.vimeo.com/video/289701084/fallback?js&referrer=https%3A%2F%2Fsketchfab.com%2F
    at net.sourceforge.htmlunit.corejs.javascript.Context.throwAsScriptRuntimeEx(Context.java:1999)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.doProcessPostponedActions(JavaScriptEngine.java:948)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.access$200(JavaScriptEngine.java:104)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine$HtmlUnitContextAction.run(JavaScriptEngine.java:887)
    ... 41 more
Caused by: com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException: 404 Not Found for https://player.vimeo.com/video/289701084/fallback?js&referrer=https%3A%2F%2Fsketchfab.com%2F
    at com.gargoylesoftware.htmlunit.WebClient.throwFailingHttpStatusCodeExceptionIfNecessary(WebClient.java:592)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:408)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:315)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement.loadInnerPageIfPossible(BaseFrameElement.java:184)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement.loadInnerPage(BaseFrameElement.java:121)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement$2.execute(BaseFrameElement.java:413)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.doProcessPostponedActions(JavaScriptEngine.java:943)
    ... 43 more
WrappedException: net.sourceforge.htmlunit.corejs.javascript.WrappedException: Wrapped com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException: 404 Not Found for https://player.vimeo.com/video/289701084/fallback?js&referrer=https%3A%2F%2Fsketchfab.com%2F
    at net.sourceforge.htmlunit.corejs.javascript.Context.throwAsScriptRuntimeEx(Context.java:1999)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.doProcessPostponedActions(JavaScriptEngine.java:948)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.access$200(JavaScriptEngine.java:104)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine$HtmlUnitContextAction.run(JavaScriptEngine.java:887)
    at net.sourceforge.htmlunit.corejs.javascript.Context.call(Context.java:616)
    at net.sourceforge.htmlunit.corejs.javascript.ContextFactory.call(ContextFactory.java:534)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.execute(JavaScriptEngine.java:772)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.execute(JavaScriptEngine.java:748)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.execute(JavaScriptEngine.java:739)
    at com.gargoylesoftware.htmlunit.html.HtmlPage.executeJavaScript(HtmlPage.java:922)
    at com.gargoylesoftware.htmlunit.html.HtmlScript.executeInlineScriptIfNeeded(HtmlScript.java:316)
    at com.gargoylesoftware.htmlunit.html.HtmlScript.executeScriptIfNeeded(HtmlScript.java:396)
    at com.gargoylesoftware.htmlunit.html.HtmlScript$2.execute(HtmlScript.java:246)
    at com.gargoylesoftware.htmlunit.html.HtmlScript.onAllChildrenAddedToPage(HtmlScript.java:267)
    at com.gargoylesoftware.htmlunit.html.HTMLParser$HtmlUnitDOMBuilder.endElement(HTMLParser.java:802)
    at org.apache.xerces.parsers.AbstractSAXParser.endElement(Unknown Source)
    at com.gargoylesoftware.htmlunit.html.HTMLParser$HtmlUnitDOMBuilder.endElement(HTMLParser.java:758)
    at net.sourceforge.htmlunit.cyberneko.HTMLTagBalancer.callEndElement(HTMLTagBalancer.java:1194)
    at net.sourceforge.htmlunit.cyberneko.HTMLTagBalancer.endElement(HTMLTagBalancer.java:1134)
    at net.sourceforge.htmlunit.cyberneko.filters.DefaultFilter.endElement(DefaultFilter.java:221)
    at net.sourceforge.htmlunit.cyberneko.filters.NamespaceBinder.endElement(NamespaceBinder.java:314)
    at net.sourceforge.htmlunit.cyberneko.HTMLScanner$ContentScanner.scanEndElement(HTMLScanner.java:3179)
    at net.sourceforge.htmlunit.cyberneko.HTMLScanner$ContentScanner.scan(HTMLScanner.java:2132)
    at net.sourceforge.htmlunit.cyberneko.HTMLScanner.scanDocument(HTMLScanner.java:939)
    at net.sourceforge.htmlunit.cyberneko.HTMLConfiguration.parse(HTMLConfiguration.java:452)
    at net.sourceforge.htmlunit.cyberneko.HTMLConfiguration.parse(HTMLConfiguration.java:403)
    at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
    at com.gargoylesoftware.htmlunit.html.HTMLParser$HtmlUnitDOMBuilder.parse(HTMLParser.java:1001)
    at com.gargoylesoftware.htmlunit.html.HTMLParser.parse(HTMLParser.java:250)
    at com.gargoylesoftware.htmlunit.html.HTMLParser.parseHtml(HTMLParser.java:196)
    at com.gargoylesoftware.htmlunit.DefaultPageCreator.createHtmlPage(DefaultPageCreator.java:267)
    at com.gargoylesoftware.htmlunit.DefaultPageCreator.createPage(DefaultPageCreator.java:158)
    at com.gargoylesoftware.htmlunit.WebClient.loadWebResponseInto(WebClient.java:531)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:398)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:315)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement.loadInnerPageIfPossible(BaseFrameElement.java:184)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement.loadInnerPage(BaseFrameElement.java:121)
    at com.gargoylesoftware.htmlunit.html.HtmlPage.loadFrames(HtmlPage.java:1894)
    at com.gargoylesoftware.htmlunit.html.HtmlPage.initialize(HtmlPage.java:251)
    at com.gargoylesoftware.htmlunit.WebClient.loadWebResponseInto(WebClient.java:538)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:398)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:315)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:466)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:448)
    at Main.main(Main.java:19)
Caused by: com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException: 404 Not Found for https://player.vimeo.com/video/289701084/fallback?js&referrer=https%3A%2F%2Fsketchfab.com%2F
    at com.gargoylesoftware.htmlunit.WebClient.throwFailingHttpStatusCodeExceptionIfNecessary(WebClient.java:592)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:408)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:315)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement.loadInnerPageIfPossible(BaseFrameElement.java:184)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement.loadInnerPage(BaseFrameElement.java:121)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement$2.execute(BaseFrameElement.java:413)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.doProcessPostponedActions(JavaScriptEngine.java:943)
    ... 43 more
Inside wrapped exception:
com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException: 404 Not Found for https://player.vimeo.com/video/289701084/fallback?js&referrer=https%3A%2F%2Fsketchfab.com%2F
    at com.gargoylesoftware.htmlunit.WebClient.throwFailingHttpStatusCodeExceptionIfNecessary(WebClient.java:592)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:408)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:315)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement.loadInnerPageIfPossible(BaseFrameElement.java:184)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement.loadInnerPage(BaseFrameElement.java:121)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement$2.execute(BaseFrameElement.java:413)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.doProcessPostponedActions(JavaScriptEngine.java:943)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.access$200(JavaScriptEngine.java:104)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine$HtmlUnitContextAction.run(JavaScriptEngine.java:887)
    at net.sourceforge.htmlunit.corejs.javascript.Context.call(Context.java:616)
    at net.sourceforge.htmlunit.corejs.javascript.ContextFactory.call(ContextFactory.java:534)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.execute(JavaScriptEngine.java:772)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.execute(JavaScriptEngine.java:748)
    at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.execute(JavaScriptEngine.java:739)
    at com.gargoylesoftware.htmlunit.html.HtmlPage.executeJavaScript(HtmlPage.java:922)
    at com.gargoylesoftware.htmlunit.html.HtmlScript.executeInlineScriptIfNeeded(HtmlScript.java:316)
    at com.gargoylesoftware.htmlunit.html.HtmlScript.executeScriptIfNeeded(HtmlScript.java:396)
    at com.gargoylesoftware.htmlunit.html.HtmlScript$2.execute(HtmlScript.java:246)
    at com.gargoylesoftware.htmlunit.html.HtmlScript.onAllChildrenAddedToPage(HtmlScript.java:267)
    at com.gargoylesoftware.htmlunit.html.HTMLParser$HtmlUnitDOMBuilder.endElement(HTMLParser.java:802)
    at org.apache.xerces.parsers.AbstractSAXParser.endElement(Unknown Source)
    at com.gargoylesoftware.htmlunit.html.HTMLParser$HtmlUnitDOMBuilder.endElement(HTMLParser.java:758)
    at net.sourceforge.htmlunit.cyberneko.HTMLTagBalancer.callEndElement(HTMLTagBalancer.java:1194)
    at net.sourceforge.htmlunit.cyberneko.HTMLTagBalancer.endElement(HTMLTagBalancer.java:1134)
    at net.sourceforge.htmlunit.cyberneko.filters.DefaultFilter.endElement(DefaultFilter.java:221)
    at net.sourceforge.htmlunit.cyberneko.filters.NamespaceBinder.endElement(NamespaceBinder.java:314)
    at net.sourceforge.htmlunit.cyberneko.HTMLScanner$ContentScanner.scanEndElement(HTMLScanner.java:3179)
    at net.sourceforge.htmlunit.cyberneko.HTMLScanner$ContentScanner.scan(HTMLScanner.java:2132)
    at net.sourceforge.htmlunit.cyberneko.HTMLScanner.scanDocument(HTMLScanner.java:939)
    at net.sourceforge.htmlunit.cyberneko.HTMLConfiguration.parse(HTMLConfiguration.java:452)
    at net.sourceforge.htmlunit.cyberneko.HTMLConfiguration.parse(HTMLConfiguration.java:403)
    at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
    at com.gargoylesoftware.htmlunit.html.HTMLParser$HtmlUnitDOMBuilder.parse(HTMLParser.java:1001)
    at com.gargoylesoftware.htmlunit.html.HTMLParser.parse(HTMLParser.java:250)
    at com.gargoylesoftware.htmlunit.html.HTMLParser.parseHtml(HTMLParser.java:196)
    at com.gargoylesoftware.htmlunit.DefaultPageCreator.createHtmlPage(DefaultPageCreator.java:267)
    at com.gargoylesoftware.htmlunit.DefaultPageCreator.createPage(DefaultPageCreator.java:158)
    at com.gargoylesoftware.htmlunit.WebClient.loadWebResponseInto(WebClient.java:531)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:398)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:315)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement.loadInnerPageIfPossible(BaseFrameElement.java:184)
    at com.gargoylesoftware.htmlunit.html.BaseFrameElement.loadInnerPage(BaseFrameElement.java:121)
    at com.gargoylesoftware.htmlunit.html.HtmlPage.loadFrames(HtmlPage.java:1894)
    at com.gargoylesoftware.htmlunit.html.HtmlPage.initialize(HtmlPage.java:251)
    at com.gargoylesoftware.htmlunit.WebClient.loadWebResponseInto(WebClient.java:538)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:398)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:315)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:466)
    at com.gargoylesoftware.htmlunit.WebClient.getPage(WebClient.java:448)
    at Main.main(Main.java:19)
======= EXCEPTION END ========

I can't deduce what the real problem is, although I can see the 404 error says it cannot find https://player.vimeo.com/... and I don't know what it has in common with https://sketchfab.com/. What is the real problem and how can I fix it?

Elgirhath
  • 409
  • 1
  • 7
  • 15

1 Answers1

1

You may want to consider using the Sketchfab API.

To resolve your error, you need to enable full HTTP request logging and compare the request headers that your library is generating with that from a major browser using the Developer Tools. Vimeo has deployed anti-scraping measures. It looks specifically like they check the referrer and may block specific user-agents.

You can reproduce the errors on the command line with curl:

curl -Li -A '' "https://player.vimeo.com/video/289701084/fallback?js&referrer=https%3A%2F%2Fsketchfab.com%2F"

Compare that with the request copied out of the browser:

curl 'https://player.vimeo.com/video/289701084?title=0&byline=0&portrait=0&transparent=0&loop=1' \
    -H 'Connection: keep-alive' \
    -H 'Upgrade-Insecure-Requests: 1' \
    -H 'DNT: 1' \
    -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36' \
    -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' \
    -H 'Referer: https://player.vimeo.com/' \
    -H 'Accept-Encoding: gzip, deflate, br' \
    -H 'Accept-Language: en-US,en;q=0.9,es;q=0.8' \
    -H 'Cookie: vuid=189212841.2077745168; continuous_play_v3=1' \
    --compressed
Gardner Bickford
  • 1,934
  • 1
  • 17
  • 25