From 6f3f0069a4d30161819a54122cd29c0ecb9eb949 Mon Sep 17 00:00:00 2001 From: goodwin Date: Sun, 28 Jan 2024 01:37:06 +0300 Subject: [PATCH] Rewrite to simple thread implementation --- src/main/java/org/example/LoadGameTask.java | 44 +++++++ .../java/org/example/RedisServiceImpl.java | 81 ++++++++++++ src/main/java/org/example/SteamConstants.java | 15 +++ .../java/org/example/SteamPriceParser.java | 122 +----------------- 4 files changed, 145 insertions(+), 117 deletions(-) create mode 100644 src/main/java/org/example/LoadGameTask.java create mode 100644 src/main/java/org/example/RedisServiceImpl.java create mode 100644 src/main/java/org/example/SteamConstants.java diff --git a/src/main/java/org/example/LoadGameTask.java b/src/main/java/org/example/LoadGameTask.java new file mode 100644 index 0000000..9b19407 --- /dev/null +++ b/src/main/java/org/example/LoadGameTask.java @@ -0,0 +1,44 @@ +package org.example; + +import com.fasterxml.jackson.databind.JsonNode; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.JedisPooled; + +import java.io.IOException; + +public class LoadGameTask implements Runnable +{ + @Override + public void run() + { + JsonNode json; + + try (JedisPooled jedis = new JedisPooled(new HostAndPort(SteamConstants.REDIS_HOST, 6380), RedisServiceImpl.prepareRedisConfig())) + { + for (int i = 0; i < 150000; i++) + { + try + { + json = RedisServiceImpl.loadGameJson(String.format(SteamConstants.GAME_DETAILS_API_TEMPLATE, i, "KZ")); + } + catch (IOException e) + { + //TODO: add pushing logs to kibana or etc.. + e.printStackTrace(); + continue; + } + //TODO: add filter for successful and unsuccessful requests + jedis.set(String.valueOf(i), json.toString()); + System.out.println("Current id = " + i); + System.out.println(json.toPrettyString()); + + System.out.println("jedis value for " + i + "\n" + jedis.get(String.valueOf(i))); + } + } + catch (Exception ex) + { + //TODO: add pushing logs to kibana or etc.. + ex.printStackTrace(); + } + } +} diff --git a/src/main/java/org/example/RedisServiceImpl.java b/src/main/java/org/example/RedisServiceImpl.java new file mode 100644 index 0000000..ada1271 --- /dev/null +++ b/src/main/java/org/example/RedisServiceImpl.java @@ -0,0 +1,81 @@ +package org.example; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import redis.clients.jedis.DefaultJedisClientConfig; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManagerFactory; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.*; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.GeneralSecurityException; +import java.security.KeyStore; + +//TODO: Add bean annotation +public class RedisServiceImpl +{ + //TODO: remove deprecated URL constructor + //TODO: replace mapper towards concrete class model GameInfoResponse + public static JsonNode loadGameJson(String url) throws IOException + { + InputStream inputStream; + + Proxy proxy = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress(SteamConstants.PROXY_HOST, SteamConstants.PROXY_PORT)); + Authenticator authenticator = new Authenticator() + { + public PasswordAuthentication getPasswordAuthentication() + { + return (new PasswordAuthentication(SteamConstants.PROXY_USER, SteamConstants.PROXY_PASS.toCharArray())); + } + }; + Authenticator.setDefault(authenticator); + URLConnection connection = new URL(url).openConnection(proxy); + inputStream = connection.getInputStream(); + + InputStreamReader reader = new InputStreamReader(inputStream); + ObjectMapper mapper = new ObjectMapper(); + return mapper.readTree(reader); + + } + + public static DefaultJedisClientConfig prepareRedisConfig() throws GeneralSecurityException, IOException + { + //Hack for building absolute path for user certificate + String keystorePath = new File("src/main/resources/redis-keystore.p12").getAbsolutePath(); + + //Hack for building absolute path for CA certificate + String truststorePath = new File("src/main/resources/truststore.jks").getAbsolutePath(); + + SSLSocketFactory sslFactory = createSslSocketFactory(truststorePath, keystorePath); + + return DefaultJedisClientConfig.builder().ssl(true).sslSocketFactory(sslFactory).user("default").password(SteamConstants.REDIS_PASS).build(); + } + + private static SSLSocketFactory createSslSocketFactory(String caCertPath, String userCertPath) throws IOException, GeneralSecurityException + { + + KeyStore keyStore = KeyStore.getInstance("pkcs12"); + keyStore.load(Files.newInputStream(Paths.get(userCertPath)), SteamConstants.USER_CERT_PASS.toCharArray()); + + KeyStore trustStore = KeyStore.getInstance("jks"); + trustStore.load(Files.newInputStream(Paths.get(caCertPath)), SteamConstants.CA_CERT_PASS.toCharArray()); + + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance("X509"); + trustManagerFactory.init(trustStore); + + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX"); + keyManagerFactory.init(keyStore, SteamConstants.USER_CERT_PASS.toCharArray()); + + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(), null); + + return sslContext.getSocketFactory(); + } +} diff --git a/src/main/java/org/example/SteamConstants.java b/src/main/java/org/example/SteamConstants.java new file mode 100644 index 0000000..364ee58 --- /dev/null +++ b/src/main/java/org/example/SteamConstants.java @@ -0,0 +1,15 @@ +package org.example; + +//TODO: move const to resources +public class SteamConstants +{ + final static String GAME_DETAILS_API_TEMPLATE = "https://store.steampowered.com/api/packagedetails?packageids=%s&cc=%s&filters=price_overview"; + final static String REDIS_HOST = "rc1a-67kduigjscq90cna.mdb.yandexcloud.net"; + final static String REDIS_PASS = "Q1598520q"; + final static String PROXY_HOST = "91.147.123.214"; + final static int PROXY_PORT = 50101; + final static String PROXY_USER = "pavel0RG3T"; + final static String PROXY_PASS = "pJ9KTE9QbD"; + final static String CA_CERT_PASS = "Q1598520q"; + final static String USER_CERT_PASS = "Q1598520q"; +} diff --git a/src/main/java/org/example/SteamPriceParser.java b/src/main/java/org/example/SteamPriceParser.java index 351cbeb..7fc7061 100644 --- a/src/main/java/org/example/SteamPriceParser.java +++ b/src/main/java/org/example/SteamPriceParser.java @@ -1,128 +1,16 @@ package org.example; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import redis.clients.jedis.DefaultJedisClientConfig; -import redis.clients.jedis.HostAndPort; -import redis.clients.jedis.JedisPooled; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; -import javax.net.ssl.KeyManagerFactory; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManagerFactory; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.*; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.GeneralSecurityException; -import java.security.KeyStore; - -//TODO: move const to resources public class SteamPriceParser { - - final static String GAME_DETAILS_API_TEMPLATE = "https://store.steampowered.com/api/packagedetails?packageids=%s&cc=%s&filters=price_overview"; - final static String REDIS_HOST = "rc1a-67kduigjscq90cna.mdb.yandexcloud.net"; - final static String REDIS_PASS = "Q1598520q"; - final static String PROXY_HOST = "91.147.123.214"; - final static int PROXY_PORT = 50101; - final static String PROXY_USER = "pavel0RG3T"; - final static String PROXY_PASS = "pJ9KTE9QbD"; - final static String CA_CERT_PASS = "Q1598520q"; - final static String USER_CERT_PASS = "Q1598520q"; - - - public static void main(String[] args) throws GeneralSecurityException, IOException + public static void main(String[] args) { - JsonNode json; - - try (JedisPooled jedis = new JedisPooled(new HostAndPort(REDIS_HOST, 6380), prepareRedisConfig())) + try (ExecutorService executorService = Executors.newFixedThreadPool(50)) { - //TODO: add threads - for (int i = 0; i < 150000; i++) - { - try - { - json = loadGameJson(String.format(GAME_DETAILS_API_TEMPLATE, i, "KZ")); - } - catch (IOException e) - { - //TODO: add pushing logs to kibana or etc.. - e.printStackTrace(); - continue; - } - //TODO: add filter for successful and unsuccessful requests - jedis.set(String.valueOf(i), json.toString()); - System.out.println("Current id = " + i); - System.out.println(json.toPrettyString()); - - System.out.println("jedis value for " + i + "\n" + jedis.get(String.valueOf(i))); - } + executorService.submit(new LoadGameTask()); } - - } - - - //TODO: move hostname, port, log and pass to resources - //TODO: remove deprecated URL constructor - //TODO: replace mapper towards concrete class model GameInfoResponse - private static JsonNode loadGameJson(String url) throws IOException - { - InputStream inputStream; - - Proxy proxy = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress(PROXY_HOST, PROXY_PORT)); - Authenticator authenticator = new Authenticator() - { - public PasswordAuthentication getPasswordAuthentication() - { - return (new PasswordAuthentication(PROXY_USER, PROXY_PASS.toCharArray())); - } - }; - Authenticator.setDefault(authenticator); - URLConnection connection = new URL(url).openConnection(proxy); - inputStream = connection.getInputStream(); - - InputStreamReader reader = new InputStreamReader(inputStream); - ObjectMapper mapper = new ObjectMapper(); - return mapper.readTree(reader); - - } - - public static DefaultJedisClientConfig prepareRedisConfig() throws GeneralSecurityException, IOException - { - //Hack for building absolute path for user certificate - String keystorePath = new File("src/main/resources/redis-keystore.p12").getAbsolutePath(); - - //Hack for building absolute path for CA certificate - String truststorePath = new File("src/main/resources/truststore.jks").getAbsolutePath(); - - SSLSocketFactory sslFactory = createSslSocketFactory(truststorePath, keystorePath); - - return DefaultJedisClientConfig.builder().ssl(true).sslSocketFactory(sslFactory).user("default").password(REDIS_PASS).build(); - } - - private static SSLSocketFactory createSslSocketFactory(String caCertPath, String userCertPath) throws IOException, GeneralSecurityException - { - - KeyStore keyStore = KeyStore.getInstance("pkcs12"); - keyStore.load(Files.newInputStream(Paths.get(userCertPath)), USER_CERT_PASS.toCharArray()); - - KeyStore trustStore = KeyStore.getInstance("jks"); - trustStore.load(Files.newInputStream(Paths.get(caCertPath)), CA_CERT_PASS.toCharArray()); - - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance("X509"); - trustManagerFactory.init(trustStore); - - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX"); - keyManagerFactory.init(keyStore, USER_CERT_PASS.toCharArray()); - - SSLContext sslContext = SSLContext.getInstance("TLS"); - sslContext.init(keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(), null); - - return sslContext.getSocketFactory(); }