diff --git a/pom.xml b/pom.xml index 499e6ca..fa8773c 100644 --- a/pom.xml +++ b/pom.xml @@ -12,6 +12,8 @@ UTF-8 + 21 + 21 @@ -55,8 +57,8 @@ org.apache.maven.plugins maven-compiler-plugin - 9 - 9 + 21 + 21 diff --git a/src/main/java/org/example/LoadGameService.java b/src/main/java/org/example/LoadGameService.java new file mode 100644 index 0000000..2a820ee --- /dev/null +++ b/src/main/java/org/example/LoadGameService.java @@ -0,0 +1,59 @@ +package org.example; + +import com.fasterxml.jackson.databind.JsonNode; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.JedisPooled; + +import java.io.IOException; +import java.security.GeneralSecurityException; + +@Service +public class LoadGameService +{ + @Autowired + private RedisService redisService; + + + public void loadGamesbySingleThread() throws InterruptedException + { + try (JedisPooled jedis = new JedisPooled(new HostAndPort(SteamConstants.REDIS_HOST, 6380), redisService.prepareRedisConfig())) + { + + JsonNode json; + int i = 0; + while (i < 150000) + { + try + { + json = redisService.loadGameJson(String.format(SteamConstants.GAME_DETAILS_API_TEMPLATE, i, "KZ")); + } + catch (IOException e) + { + //TODO: add pushing logs to kibana or etc.. + e.printStackTrace(); + if (e.getMessage().equals("429")) + { + Thread.sleep(30000); + continue; + } + i++; + continue; + + } + //TODO: add filter for successful and unsuccessful requests + jedis.set(String.valueOf(i), json.toString()); + System.out.println("Current id = " + i); + System.out.println(json.toPrettyString()); + i++; + } + + } + catch (GeneralSecurityException | IOException ex) + { + //TODO: add pushing logs to kibana or etc.. + ex.printStackTrace(); + } + } +} diff --git a/src/main/java/org/example/LoadGameTask.java b/src/main/java/org/example/LoadGameTask.java index 12e65e0..40f7adf 100644 --- a/src/main/java/org/example/LoadGameTask.java +++ b/src/main/java/org/example/LoadGameTask.java @@ -12,32 +12,48 @@ import java.io.IOException; public class LoadGameTask implements Runnable { @Autowired - private RedisServiceImpl redisService; + private RedisService redisService; @Override public void run() { JsonNode json; + int i = 0; + while (true) + { + if (SteamPriceParser.gameCodeList.get(i) == null) + { + break; + } + if (!SteamPriceParser.gameCodeList.get(i)) + { + //Block item + SteamPriceParser.gameCodeList.put(i, true); + break; + } + } + try (JedisPooled jedis = new JedisPooled(new HostAndPort(SteamConstants.REDIS_HOST, 6380), redisService.prepareRedisConfig())) { - for (int i = 0; i < 150000; i++) + try { - try - { - json = redisService.loadGameJson(String.format(SteamConstants.GAME_DETAILS_API_TEMPLATE, i, "KZ")); - } - catch (IOException e) - { - //TODO: add pushing logs to kibana or etc.. - e.printStackTrace(); - continue; - } - //TODO: add filter for successful and unsuccessful requests - jedis.set(String.valueOf(i), json.toString()); - System.out.println("Current id = " + i); - System.out.println(json.toPrettyString()); + json = redisService.loadGameJson(String.format(SteamConstants.GAME_DETAILS_API_TEMPLATE, i, "KZ")); } + catch (IOException e) + { + //TODO: add pushing logs to kibana or etc.. + e.printStackTrace(); + unblockItem(i); + return; + + } + //TODO: add filter for successful and unsuccessful requests + jedis.set(String.valueOf(i), json.toString()); + System.out.println("Current id = " + i); + System.out.println(json.toPrettyString()); + removeItem(i); + } catch (Exception ex) { @@ -45,4 +61,14 @@ public class LoadGameTask implements Runnable ex.printStackTrace(); } } + + private synchronized void unblockItem(int i) + { + SteamPriceParser.gameCodeList.put(i, false); + } + + private synchronized void removeItem(int i) + { + SteamPriceParser.gameCodeList.remove(i); + } } diff --git a/src/main/java/org/example/RedisServiceImpl.java b/src/main/java/org/example/RedisService.java similarity index 91% rename from src/main/java/org/example/RedisServiceImpl.java rename to src/main/java/org/example/RedisService.java index ce88e3c..0b7cecf 100644 --- a/src/main/java/org/example/RedisServiceImpl.java +++ b/src/main/java/org/example/RedisService.java @@ -20,7 +20,7 @@ import java.security.GeneralSecurityException; import java.security.KeyStore; @Service -public class RedisServiceImpl +public class RedisService { //TODO: remove deprecated URL constructor //TODO: replace mapper towards concrete class model GameInfoResponse @@ -38,7 +38,15 @@ public class RedisServiceImpl }; Authenticator.setDefault(authenticator); URLConnection connection = new URL(url).openConnection(proxy); - inputStream = connection.getInputStream(); + try + { + inputStream = connection.getInputStream(); + } + catch (IOException exception) + { + int statusCode = ((HttpURLConnection)connection).getResponseCode(); + throw new IOException(String.valueOf(statusCode)); + } InputStreamReader reader = new InputStreamReader(inputStream); ObjectMapper mapper = new ObjectMapper(); diff --git a/src/main/java/org/example/SteamPriceParser.java b/src/main/java/org/example/SteamPriceParser.java index 43fdaa2..824b70a 100644 --- a/src/main/java/org/example/SteamPriceParser.java +++ b/src/main/java/org/example/SteamPriceParser.java @@ -1,9 +1,11 @@ package org.example; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.AnnotationConfigApplicationContext; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -11,15 +13,28 @@ import java.util.concurrent.Executors; @ComponentScan public class SteamPriceParser { - public static void main(String[] args) + + + public static Map gameCodeList = new HashMap<>(); + + static { + for (int i = 0; i < 150000; i++) + { + gameCodeList.put(i, false); + } + } + public static void main(String[] args) throws InterruptedException + { +// AnnotationConfigApplicationContext applicationContext = new AnnotationConfigApplicationContext(SteamPriceParser.class); +// ExecutorService executorService = Executors.newFixedThreadPool(50); +// +// // Hack for adding runnable to spring watching list +// LoadGameTask task = applicationContext.getBean(LoadGameTask.class); +// executorService.submit(task); AnnotationConfigApplicationContext applicationContext = new AnnotationConfigApplicationContext(SteamPriceParser.class); - - ExecutorService executorService = Executors.newFixedThreadPool(50); - - // Hack for adding runnable to spring watching list - LoadGameTask task = applicationContext.getBean(LoadGameTask.class); - executorService.submit(task); + LoadGameService loadGameService = applicationContext.getBean(LoadGameService.class); + loadGameService.loadGamesbySingleThread(); } }