0

I try to open stream to this url: https://www.google.com/search?q=site:foxnews.com&tbas=0&tbs=qdr:d&tbm=nws&tbas=0&source=lnt&sa=X&ved=0ahUKEwiY6On-1L3eAhUBQhoKHb2rBfIQpwUIHw&biw=1540&bih=832&dpr=1.09 but it returns an IOException. I suspect that it has something to do with the params because when I remove them it returns the requested page but not the correct one because of the missing params. here is my code.

public List<String> findUrls(String url) {

    List<String> urls = null;
    InputStream response = null;
    try {
        response = new URL(url).openStream();

        System.err.println("opened stream");

        @SuppressWarnings("resource")
        Scanner scanner = new Scanner(response);
        String responseBody = scanner.useDelimiter("\\A").next();

        urls = getUrls(responseBody);

    } catch (IOException ex) {
        ex.printStackTrace();
        System.err.println("Error finding articles");
    } finally {
        try {
            response.close();
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }

    return urls;
}

@Scheduled(fixedDelay = 86400000)
public void autoUpload() {

    System.out.println("Method executed at every 24 hours. Current time is :: " + new Date());

    List<String> urls = findUrls(
            "https://www.google.com/search?q=site:foxnews.com&tbas=0&tbs=qdr:d&tbm=nws&tbas=0&source=lnt&sa=X&ved=0ahUKEwiY6On-1L3eAhUBQhoKHb2rBfIQpwUIHw&biw=1540&bih=832&dpr=1.09");
    createArticles(urls);
}

Here is the stacktrace:

java.io.IOException: Server returned HTTP response code: 403 for URL: https://www.google.com/search?q=site:foxnews.com&tbas=0&tbs=qdr:d&tbm=nws&tbas=0&source=lnt&sa=X&ved=0ahUKEwiY6On-1L3eAhUBQhoKHb2rBfIQpwUIHw&biw=1540&bih=832&dpr=1.09
    at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(Unknown Source)
    at sun.net.www.protocol.http.HttpURLConnection.getInputStream(Unknown Source)
    at sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(Unknown Source)
    at java.net.URL.openStream(Unknown Source)
    at com.debugger.spring.web.service.NewsService.findUrls(NewsService.java:531)
    at com.debugger.spring.web.service.NewsService$$FastClassBySpringCGLIB$$14257c0e.invoke(<generated>)
    at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
    at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:684)
    at com.debugger.spring.web.service.NewsService$$EnhancerBySpringCGLIB$$59deebe3.findUrls(<generated>)
    at com.debugger.spring.web.tests.UploadArticleTest.testAutoUpload(UploadArticleTest.java:592)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
    at java.lang.reflect.Method.invoke(Unknown Source)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.springframework.test.context.junit4.statements.RunBeforeTestMethodCallbacks.evaluate(RunBeforeTestMethodCallbacks.java:74)
    at org.springframework.test.context.junit4.statements.RunAfterTestMethodCallbacks.evaluate(RunAfterTestMethodCallbacks.java:85)
    at org.springframework.test.context.junit4.statements.SpringRepeat.evaluate(SpringRepeat.java:86)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
    at org.springframework.test.context.junit4.SpringJUnit4ClassRunner.runChild(SpringJUnit4ClassRunner.java:241)
    at org.springframework.test.context.junit4.SpringJUnit4ClassRunner.runChild(SpringJUnit4ClassRunner.java:87)
    at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
    at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
    at org.springframework.test.context.junit4.statements.RunBeforeTestClassCallbacks.evaluate(RunBeforeTestClassCallbacks.java:61)
    at org.springframework.test.context.junit4.statements.RunAfterTestClassCallbacks.evaluate(RunAfterTestClassCallbacks.java:70)
    at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
    at org.springframework.test.context.junit4.SpringJUnit4ClassRunner.run(SpringJUnit4ClassRunner.java:180)
    at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:86)
    at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:538)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:760)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:460)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:206)
DisplayName
  • 69
  • 2
  • 9
  • What, exactly, returns null? openStream() cannot return null. – VGR Nov 05 '18 at 20:34
  • It returns an IOException, the point is it doesn't return a stream. – DisplayName Nov 06 '18 at 10:20
  • Edit your question and include the *entire stack trace* of that IOException, including any `Caused by:` sections. That exception is telling you exactly what went wrong and why. – VGR Nov 06 '18 at 14:30
  • I added the exception but I cannot understand why access is denied. – DisplayName Nov 06 '18 at 16:21
  • The HTTP 403 error means that you do not have access to the resource. You can read up here: https://en.wikipedia.org/wiki/HTTP_403 – JPadley Nov 06 '18 at 16:24
  • I have fixed the problem. It turns out I had to "fool" the system. This is what I did:HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); conn.addRequestProperty("User-Agent", "Mozilla/4.0"); response = conn.getInputStream(); – DisplayName Nov 06 '18 at 17:04

0 Answers0