Skip to content
TheChatArchiveThe Chat Archive
gpt-3.5-turbo-0613· OpenAIAi2 WildChat-4.8M (AI2)

java code for websearch with duckduckgo with links as results, take care of mimic…

  1. User

    java code for websearch with duckduckgo with links as results, take care of mimic mozilla agent

  2. Assistant

    Sure, here's a Java code example for performing a web search with DuckDuckGo using their API and setting the user-agent to mimic Mozilla.

    import java.io.BufferedReader;
    import java.io.InputStreamReader;
    import java.net.HttpURLConnection;
    import java.net.URL;
    import java.net.URLEncoder;
    
    public class DuckDuckGoWebSearch {
        public static void main(String[] args) {
            try {
                // Set the search query
                String query = "java programming";
    
                // URL-encode the query
                String encodedQuery = URLEncoder.encode(query, "UTF-8");
    
                // Create the API URL with encoded query
                String apiUrl = "https://api.duckduckgo.com/?q=" + encodedQuery + "&format=json";
    
                // Create a URL object from the API URL string
                URL url = new URL(apiUrl);
    
                // Create a HttpURLConnection object and set the user-agent header
                HttpURLConnection connection = (HttpURLConnection) url.openConnection();
                connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3");
    
                // Send a GET request to the API
                connection.setRequestMethod("GET");
    
                // Get the response code
                int responseCode = connection.getResponseCode();
                if (responseCode == HttpURLConnection.HTTP_OK) {
                    // Read the response
                    BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
                    StringBuilder response = new StringBuilder();
                    String line;
                    while ((line = reader.readLine()) != null) {
                        response.append(line);
                    }
                    reader.close();
    
                    // Parse the JSON response
                    String jsonString = response.toString();
                    // ... (parse the JSON and extract the search results)
    
                    // Print the search results
                    System.out.println(jsonString);
                } else {
                    System.out.println("Request failed with response code: " + responseCode);
                }
    
                // Cleanup - close the connection
                connection.disconnect();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
    

    Please note that in this code, we are calling the DuckDuckGo Instant Answer API with the query as a parameter. You will need to parse the JSON response and extract the search results based on your specific requirements.

  3. User

    same code but with jsoup parsing the result

  4. Assistant

    Certainly! Here's an updated version of the code that uses Jsoup library to parse the JSON response and extract the search results.

    import org.jsoup.Jsoup;
    import org.jsoup.nodes.Document;
    import org.jsoup.nodes.Element;
    import org.jsoup.select.Elements;
    
    import java.io.BufferedReader;
    import java.io.InputStreamReader;
    import java.net.HttpURLConnection;
    import java.net.URL;
    import java.net.URLEncoder;
    
    public class DuckDuckGoWebSearch {
        public static void main(String[] args) {
            try {
                // Set the search query
                String query = "java programming";
    
                // URL-encode the query
                String encodedQuery = URLEncoder.encode(query, "UTF-8");
    
                // Create the API URL with encoded query
                String apiUrl = "https://api.duckduckgo.com/?q=" + encodedQuery + "&format=json";
    
                // Create a URL object from the API URL string
                URL url = new URL(apiUrl);
    
                // Create a HttpURLConnection object and set the user-agent header
                HttpURLConnection connection = (HttpURLConnection) url.openConnection();
                connection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3");
    
                // Send a GET request to the API
                connection.setRequestMethod("GET");
    
                // Get the response code
                int responseCode = connection.getResponseCode();
                if (responseCode == HttpURLConnection.HTTP_OK) {
                    // Read the response
                    BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
                    StringBuilder response = new StringBuilder();
                    String line;
                    while ((line = reader.readLine()) != null) {
                        response.append(line);
                    }
                    reader.close();
    
                    // Parse the JSON response with Jsoup
                    String jsonString = response.toString();
                    Document doc = Jsoup.parse(jsonString);
    
                    // Extract the search results using Jsoup selectors
                    Elements results = doc.select("Results > Result");
                    for (Element result : results) {
                        String title = result.selectFirst("Text").text();
                        String urlResult = result.selectFirst("FirstURL").text();
    
                        System.out.println("Title: " + title);
                        System.out.println("URL: " + urlResult);
                        System.out.println();
                    }
                } else {
                    System.out.println("Request failed with response code: " + responseCode);
                }
    
                // Cleanup - close the connection
                connection.disconnect();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
    

    In this updated version, after obtaining the JSON response, it is parsed with Jsoup's parse method, and then the search results are extracted using the appropriate Jsoup selectors (Results > Result). The title and URL of each search result are printed to the console.