I try to open stream to this url: https://www.google.com/search?q=site:foxnews.com&tbas=0&tbs=qdr:d&tbm=nws&tbas=0&source=lnt&sa=X&ved=0ahUKEwiY6On-1L3eAhUBQhoKHb2rBfIQpwUIHw&biw=1540&bih=832&dpr=1.09 but it returns an IOException. I suspect that it has something to do with the params because when I remove them it returns the requested page but not the correct one because of the missing params. here is my code.
public List<String> findUrls(String url) {
List<String> urls = null;
InputStream response = null;
try {
response = new URL(url).openStream();
System.err.println("opened stream");
@SuppressWarnings("resource")
Scanner scanner = new Scanner(response);
String responseBody = scanner.useDelimiter("\\A").next();
urls = getUrls(responseBody);
} catch (IOException ex) {
ex.printStackTrace();
System.err.println("Error finding articles");
} finally {
try {
response.close();
} catch (Exception ex) {
ex.printStackTrace();
}
}
return urls;
}
@Scheduled(fixedDelay = 86400000)
public void autoUpload() {
System.out.println("Method executed at every 24 hours. Current time is :: " + new Date());
List<String> urls = findUrls(
"https://www.google.com/search?q=site:foxnews.com&tbas=0&tbs=qdr:d&tbm=nws&tbas=0&source=lnt&sa=X&ved=0ahUKEwiY6On-1L3eAhUBQhoKHb2rBfIQpwUIHw&biw=1540&bih=832&dpr=1.09");
createArticles(urls);
}
Here is the stacktrace:
java.io.IOException: Server returned HTTP response code: 403 for URL: https://www.google.com/search?q=site:foxnews.com&tbas=0&tbs=qdr:d&tbm=nws&tbas=0&source=lnt&sa=X&ved=0ahUKEwiY6On-1L3eAhUBQhoKHb2rBfIQpwUIHw&biw=1540&bih=832&dpr=1.09
at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(Unknown Source)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(Unknown Source)
at sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(Unknown Source)
at java.net.URL.openStream(Unknown Source)
at com.debugger.spring.web.service.NewsService.findUrls(NewsService.java:531)
at com.debugger.spring.web.service.NewsService$$FastClassBySpringCGLIB$$14257c0e.invoke(<generated>)
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:684)
at com.debugger.spring.web.service.NewsService$$EnhancerBySpringCGLIB$$59deebe3.findUrls(<generated>)
at com.debugger.spring.web.tests.UploadArticleTest.testAutoUpload(UploadArticleTest.java:592)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.springframework.test.context.junit4.statements.RunBeforeTestMethodCallbacks.evaluate(RunBeforeTestMethodCallbacks.java:74)
at org.springframework.test.context.junit4.statements.RunAfterTestMethodCallbacks.evaluate(RunAfterTestMethodCallbacks.java:85)
at org.springframework.test.context.junit4.statements.SpringRepeat.evaluate(SpringRepeat.java:86)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at org.springframework.test.context.junit4.SpringJUnit4ClassRunner.runChild(SpringJUnit4ClassRunner.java:241)
at org.springframework.test.context.junit4.SpringJUnit4ClassRunner.runChild(SpringJUnit4ClassRunner.java:87)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.springframework.test.context.junit4.statements.RunBeforeTestClassCallbacks.evaluate(RunBeforeTestClassCallbacks.java:61)
at org.springframework.test.context.junit4.statements.RunAfterTestClassCallbacks.evaluate(RunAfterTestClassCallbacks.java:70)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.springframework.test.context.junit4.SpringJUnit4ClassRunner.run(SpringJUnit4ClassRunner.java:180)
at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:86)
at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:538)
at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:760)
at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:460)
at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:206)