goodwin/SteamPriceParser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite to simple thread implementation

Browse Source
This commit is contained in:
goodwin
2024-01-28 01:37:06 +03:00
parent 5b9ba88867
commit 6f3f0069a4
4 changed files with 145 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/main/java/org/example/LoadGameTask.java Normal file
View File

@ -0,0 +1,44 @@
package org.example;
import com.fasterxml.jackson.databind.JsonNode;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisPooled;
import java.io.IOException;
public class LoadGameTask implements Runnable
{
@Override
public void run()
{
JsonNode json;
try (JedisPooled jedis = new JedisPooled(new HostAndPort(SteamConstants.REDIS_HOST, 6380), RedisServiceImpl.prepareRedisConfig()))
{
for (int i = 0; i < 150000; i++)
{
try
{
json = RedisServiceImpl.loadGameJson(String.format(SteamConstants.GAME_DETAILS_API_TEMPLATE, i, "KZ"));
}
catch (IOException e)
{
//TODO: add pushing logs to kibana or etc..
e.printStackTrace();
continue;
}
//TODO: add filter for successful and unsuccessful requests
jedis.set(String.valueOf(i), json.toString());
System.out.println("Current id = " + i);
System.out.println(json.toPrettyString());
System.out.println("jedis value for " + i + "\n" + jedis.get(String.valueOf(i)));
}
}
catch (Exception ex)
{
//TODO: add pushing logs to kibana or etc..
ex.printStackTrace();
}
}
}

81
src/main/java/org/example/RedisServiceImpl.java Normal file
View File

@ -0,0 +1,81 @@
package org.example;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import redis.clients.jedis.DefaultJedisClientConfig;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManagerFactory;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
//TODO: Add bean annotation
public class RedisServiceImpl
{
//TODO: remove deprecated URL constructor
//TODO: replace mapper towards concrete class model GameInfoResponse
public static JsonNode loadGameJson(String url) throws IOException
{
InputStream inputStream;
Proxy proxy = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress(SteamConstants.PROXY_HOST, SteamConstants.PROXY_PORT));
Authenticator authenticator = new Authenticator()
{
public PasswordAuthentication getPasswordAuthentication()
{
return (new PasswordAuthentication(SteamConstants.PROXY_USER, SteamConstants.PROXY_PASS.toCharArray()));
}
};
Authenticator.setDefault(authenticator);
URLConnection connection = new URL(url).openConnection(proxy);
inputStream = connection.getInputStream();
InputStreamReader reader = new InputStreamReader(inputStream);
ObjectMapper mapper = new ObjectMapper();
return mapper.readTree(reader);
}
public static DefaultJedisClientConfig prepareRedisConfig() throws GeneralSecurityException, IOException
{
//Hack for building absolute path for user certificate
String keystorePath = new File("src/main/resources/redis-keystore.p12").getAbsolutePath();
//Hack for building absolute path for CA certificate
String truststorePath = new File("src/main/resources/truststore.jks").getAbsolutePath();
SSLSocketFactory sslFactory = createSslSocketFactory(truststorePath, keystorePath);
return DefaultJedisClientConfig.builder().ssl(true).sslSocketFactory(sslFactory).user("default").password(SteamConstants.REDIS_PASS).build();
}
private static SSLSocketFactory createSslSocketFactory(String caCertPath, String userCertPath) throws IOException, GeneralSecurityException
{
KeyStore keyStore = KeyStore.getInstance("pkcs12");
keyStore.load(Files.newInputStream(Paths.get(userCertPath)), SteamConstants.USER_CERT_PASS.toCharArray());
KeyStore trustStore = KeyStore.getInstance("jks");
trustStore.load(Files.newInputStream(Paths.get(caCertPath)), SteamConstants.CA_CERT_PASS.toCharArray());
TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance("X509");
trustManagerFactory.init(trustStore);
KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX");
keyManagerFactory.init(keyStore, SteamConstants.USER_CERT_PASS.toCharArray());
SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(), null);
return sslContext.getSocketFactory();
}
}

15
src/main/java/org/example/SteamConstants.java Normal file
View File

@ -0,0 +1,15 @@
package org.example;
//TODO: move const to resources
public class SteamConstants
{
final static String GAME_DETAILS_API_TEMPLATE = "https://store.steampowered.com/api/packagedetails?packageids=%s&cc=%s&filters=price_overview";
final static String REDIS_HOST = "rc1a-67kduigjscq90cna.mdb.yandexcloud.net";
final static String REDIS_PASS = "Q1598520q";
final static String PROXY_HOST = "91.147.123.214";
final static int PROXY_PORT = 50101;
final static String PROXY_USER = "pavel0RG3T";
final static String PROXY_PASS = "pJ9KTE9QbD";
final static String CA_CERT_PASS = "Q1598520q";
final static String USER_CERT_PASS = "Q1598520q";
}

122
src/main/java/org/example/SteamPriceParser.java
View File

@ -1,128 +1,16 @@
package org.example;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import redis.clients.jedis.DefaultJedisClientConfig;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisPooled;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManagerFactory;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
//TODO: move const to resources
public class SteamPriceParser
{
final static String GAME_DETAILS_API_TEMPLATE = "https://store.steampowered.com/api/packagedetails?packageids=%s&cc=%s&filters=price_overview";
final static String REDIS_HOST = "rc1a-67kduigjscq90cna.mdb.yandexcloud.net";
final static String REDIS_PASS = "Q1598520q";
final static String PROXY_HOST = "91.147.123.214";
final static int PROXY_PORT = 50101;
final static String PROXY_USER = "pavel0RG3T";
final static String PROXY_PASS = "pJ9KTE9QbD";
final static String CA_CERT_PASS = "Q1598520q";
final static String USER_CERT_PASS = "Q1598520q";
public static void main(String[] args) throws GeneralSecurityException, IOException
public static void main(String[] args)
{
JsonNode json;
try (JedisPooled jedis = new JedisPooled(new HostAndPort(REDIS_HOST, 6380), prepareRedisConfig()))
try (ExecutorService executorService = Executors.newFixedThreadPool(50))
{
//TODO: add threads
for (int i = 0; i < 150000; i++)
{
try
{
json = loadGameJson(String.format(GAME_DETAILS_API_TEMPLATE, i, "KZ"));
}
catch (IOException e)
{
//TODO: add pushing logs to kibana or etc..
e.printStackTrace();
continue;
}
//TODO: add filter for successful and unsuccessful requests
jedis.set(String.valueOf(i), json.toString());
System.out.println("Current id = " + i);
System.out.println(json.toPrettyString());
System.out.println("jedis value for " + i + "\n" + jedis.get(String.valueOf(i)));
}
executorService.submit(new LoadGameTask());
}
}
//TODO: move hostname, port, log and pass to resources
//TODO: remove deprecated URL constructor
//TODO: replace mapper towards concrete class model GameInfoResponse
private static JsonNode loadGameJson(String url) throws IOException
{
InputStream inputStream;
Proxy proxy = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress(PROXY_HOST, PROXY_PORT));
Authenticator authenticator = new Authenticator()
{
public PasswordAuthentication getPasswordAuthentication()
{
return (new PasswordAuthentication(PROXY_USER, PROXY_PASS.toCharArray()));
}
};
Authenticator.setDefault(authenticator);
URLConnection connection = new URL(url).openConnection(proxy);
inputStream = connection.getInputStream();
InputStreamReader reader = new InputStreamReader(inputStream);
ObjectMapper mapper = new ObjectMapper();
return mapper.readTree(reader);
}
public static DefaultJedisClientConfig prepareRedisConfig() throws GeneralSecurityException, IOException
{
//Hack for building absolute path for user certificate
String keystorePath = new File("src/main/resources/redis-keystore.p12").getAbsolutePath();
//Hack for building absolute path for CA certificate
String truststorePath = new File("src/main/resources/truststore.jks").getAbsolutePath();
SSLSocketFactory sslFactory = createSslSocketFactory(truststorePath, keystorePath);
return DefaultJedisClientConfig.builder().ssl(true).sslSocketFactory(sslFactory).user("default").password(REDIS_PASS).build();
}
private static SSLSocketFactory createSslSocketFactory(String caCertPath, String userCertPath) throws IOException, GeneralSecurityException
{
KeyStore keyStore = KeyStore.getInstance("pkcs12");
keyStore.load(Files.newInputStream(Paths.get(userCertPath)), USER_CERT_PASS.toCharArray());
KeyStore trustStore = KeyStore.getInstance("jks");
trustStore.load(Files.newInputStream(Paths.get(caCertPath)), CA_CERT_PASS.toCharArray());
TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance("X509");
trustManagerFactory.init(trustStore);
KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX");
keyManagerFactory.init(keyStore, USER_CERT_PASS.toCharArray());
SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(), null);
return sslContext.getSocketFactory();
}