[Java] URL Download

1. URL To File (Redirect)

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28

public static void urlToFile_red(String fileURL, String saveFilePath) throws IOException { File file = new File(saveFilePath); if( file.getParentFile().exists() == false ) { file.getParentFile().mkdirs(); } URL url = new URL(fileURL); HttpURLConnection httpConn = (HttpURLConnection) url.openConnection(); httpConn.setConnectTimeout(30000); httpConn.setReadTimeout(30000); httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36"); int responseCode = httpConn.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { InputStream inputStream = httpConn.getInputStream(); FileOutputStream outputStream = new FileOutputStream(saveFilePath); int bytesRead = -1; byte[] buffer = new byte[1024]; while ((bytesRead = inputStream.read(buffer)) != -1) { outputStream.write(buffer, 0, bytesRead); } outputStream.close(); inputStream.close(); //System.out.println("File downloaded"); } else { System.out.println("No file to download. Server replied HTTP code: " + responseCode); } httpConn.disconnect(); }

2. URL To String (Redirect)

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35

public static String urlToString_red(String webUrl) throws IOException { HttpURLConnection httpConn; int responseCode; while (true) { URL resourceUrl = new URL(webUrl); httpConn = (HttpURLConnection) resourceUrl.openConnection(); httpConn.setConnectTimeout(30000); httpConn.setReadTimeout(30000); httpConn.setInstanceFollowRedirects(false); // Make the logic below easier to detect redirections httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36"); responseCode = httpConn.getResponseCode(); switch( responseCode ) { case HttpURLConnection.HTTP_MOVED_PERM: case HttpURLConnection.HTTP_MOVED_TEMP: String location = httpConn.getHeaderField("Location"); location = URLDecoder.decode(location, "UTF-8"); URL base = new URL(webUrl); URL next = new URL(base, location); // Deal with relative URLs webUrl = next.toExternalForm(); continue; } break; } String html = null; if (responseCode == HttpURLConnection.HTTP_OK) { InputStream inputStream = httpConn.getInputStream(); html = InputStreamToStringByRead(inputStream); } else { System.out.println("No file to download. Server replied HTTP code: " + responseCode); } httpConn.disconnect(); return html; }

3. URL To String (No Redirect)

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17

public String urlToString_nored(String webUrl) throws IOException { String html = null; URL url = new URL(webUrl); HttpURLConnection httpConn = (HttpURLConnection) url.openConnection(); httpConn.setInstanceFollowRedirects(true); httpConn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36"); int responseCode = httpConn.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { InputStream inputStream = httpConn.getInputStream(); html = InputStreamToStringByRead(inputStream); } else { System.out.println("No file to download. Server replied HTTP code: " + responseCode); } httpConn.disconnect(); return html; }

4. URL To File (Apache)

1 2 3 4 5

public static void urlToFileApache(String url, String fileName) throws MalformedURLException, IOException { int CONNECT_TIMEOUT = 30000; int READ_TIMEOUT = 30000; FileUtils.copyURLToFile(new URL(url), new File(fileName),CONNECT_TIMEOUT,READ_TIMEOUT); }