import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class ParseSite extends ProxyAdd {

    public static final int TIMEOUT = 30000; // Тайм-аут 30 секунд

    public ParseSite(String no_chek, String good_chek) {
        try (BufferedReader br = new BufferedReader(new FileReader(no_chek));
             BufferedWriter writer = new BufferedWriter(new FileWriter(good_chek, true))) {

            String line;
            while ((line = br.readLine()) != null) {
                try {
                    String url = "https://duckduckgo.com/html/?q=" + line;

                    Thread.sleep(210); // Задержка между запросами
                    Document doc = Jsoup.connect(url).timeout(TIMEOUT).get(); // Установка тайм-аута
                    Elements links = doc.select("a.result__a");

                    int count = 0;
                    for (Element link : links) {
                        String href = link.attr("href");

                        regCheck(href, writer);
                        System.out.println("Check -> " + line);
                        count++;
                        Thread.sleep(210);
                        if (count == 28) {
                            break;
                        }
                    }
                } catch (Exception e) {
                    System.err.println("Error processing line: " + line + " - " + e.getMessage());
                }
            }
        } catch (Exception e) {
            System.err.println("File error - " + e.getMessage());
        }
    }

    public ParseSite(String no_chek, String good_chek, boolean privat_proxy,
                     String ip, String port, String user, String password) {

        try (BufferedReader br = new BufferedReader(new FileReader(no_chek));
             BufferedWriter writer = new BufferedWriter(new FileWriter(good_chek, true))) {

            String line;
            while ((line = br.readLine()) != null) {
                try {
                    Document doc;
                    if (privat_proxy) {
                        doc = privateProxyParse(ip, port, user, password, line);
                    } else {
                        doc = publicProxyParse(ip, port, line);
                    }
                    Elements links = doc.select("a.result__a");

                    int count = 0;
                    for (Element link : links) {
                        String href = link.attr("href");

                        regCheck(href, writer);
                        count++;
                        Thread.sleep(210);
                        if (count == 28) {
                            break;
                        }
                    }
                } catch (Exception e) {
                    System.err.println("Error processing line: " + line + " - " + e.getMessage());
                }
            }
        } catch (Exception e) {
            System.err.println("Error - " + e.getMessage());
        }
    }

    private void regCheck(String url, BufferedWriter pw) {
        try {
            Thread.sleep(210);

            Pattern pattern = Pattern.compile("^(https?://)?(www\\.)?([a-zA-Z0-9.-]+)(/.*)?$");
            Matcher matcher = pattern.matcher(url);
            if (matcher.find()) {
                String domain = matcher.group(3);
                if (!domain.equals("github.com") && !domain.equals("adobe.com") && !domain.equals("helpx.adobe.com") &&
                        !domain.equals("community.adobe.com") && !domain.equals("reddit.com") && !domain.equals("youtube.com")
                        && !domain.equals("stackoverflow.com") && !domain.equals("habr.com") && !domain.equals("geekbrains.com")
                        && !domain.equals("help.adobe.com")  && !domain.equals("microsoft.com")  && !domain.equals("apple.com")
                        && !domain.equals("support.apple.com") && !domain.equals("binance.com")) {

                    pw.append(domain).append("\n"); // Добавляем домен в файл
                    System.out.println("Domain -> " + domain);
                }
            }
            Thread.sleep(210);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
}



















