Skip to content
The Chat Archive
gpt-3.5-turbo-0301· OpenAI WildChat-4.8M (AI2)

Act as: MERN Developer Technology stack: Node.js, Express.js, MongoDB, Mongoose, ReactJS…

  1. User

    Act as: MERN Developer

    Technology stack: Node.js, Express.js, MongoDB, Mongoose, ReactJS

    Functionality: scrape google trends and populate graph in ui

    Task: Make an API that scrape google trends data and populate graph in UI. Generate only code. Mention file names with folder structure.

  2. Assistant

    Folder Structure:

    • server
      • api
        • controllers
          • scraper.js
        • routes
          • scraper.js
      • config
        • db.js
      • models
        • trend.js
      • utils
        • scraper.js
      • app.js
    • client
      • public
        • index.html
        • index.css
      • src
        • components
          • graph.jsx
        • App.js
        • index.js

    File names and Code:

    server/api/controllers/scraper.js

    const Trend = require("../models/trend");
    const scraper = require("../utils/scraper");
    
    // Function to scrape and save trend data
    const saveTrendData = async (query) => {
      try {
        const data = await scraper.scrape(query);
        // Save trend data to database
        const trend = new Trend({ query, data });
        await trend.save();
        return true;
      } catch (error) {
        console.log(error.message);
        return false;
      }
    };
    
    // Controller to handle saving trend data
    exports.saveTrend = async (req, res) => {
      const { query } = req.body;
      const saved = await saveTrendData(query);
      if (saved) {
        res.send("Trend data saved successfully!");
      } else {
        res.status(500).send("Error occurred while saving data");
      }
    };
    
    // Function to retrieve trend data
    exports.getTrendData = async (req, res) => {
      const { query } = req.params;
      try {
        // Retrieve trend data from database
        const trend = await Trend.findOne({ query });
        res.send(trend);
      } catch (error) {
        res.status(500).send("Error occurred while retrieving data");
      }
    };
    

    server/api/routes/scraper.js

    const express = require("express");
    const scraperController = require("../controllers/scraper");
    
    const router = express.Router();
    
    // Route to save trend data to database
    router.post("/", scraperController.saveTrend);
    
    // Route to get trend data from database
    router.get("/:query", scraperController.getTrendData);
    
    module.exports = router;
    

    server/config/db.js

    const mongoose = require("mongoose");
    
    // Database connection function
    const connect = async () => {
      try {
        await mongoose.connect(MONGODB_URI, {
          useNewUrlParser: true,
          useUnifiedTopology: true,
        });
        console.log("Database connected successfully");
      } catch (error) {
        console.log(error.message);
      }
    };
    
    module.exports = { connect };
    

    server/models/trend.js

    const mongoose = require("mongoose");
    
    // Schema for trend data
    const TrendSchema = new mongoose.Schema({
      query: { type: String, required: true },
      data: { type: Object, required: true },
    });
    
    const Trend = mongoose.model("Trend", TrendSchema);
    
    module.exports = Trend;
    

    server/utils/scraper.js

    const puppeteer = require("puppeteer");
    
    // Function to scrape trend data
    exports.scrape = async (query) => {
      const browser = await puppeteer.launch();
      const page = await browser.newPage();
      await page.goto(`https://trends.google.com/trends/explore?q=${query}`);
      await page.waitForSelector(".widget-div.widget-gpt");
    
      const data = await page.evaluate(() => {
        const tableRows = Array.from(
          document.querySelectorAll(".widget-div.widget-gpt tbody tr")
        );
        return tableRows.map((row) => {
          const cells = row.querySelectorAll("td");
          return {
            date: cells[0].textContent.trim(),
            value: cells[1].textContent.trim(),
          };
        });
      });
    
      await browser.close();
      return data;
    };
    

    client/src/components/graph.jsx

    import React, { useState, useEffect } from "react";
    import axios from "axios";
    import { LineChart, CartesianGrid, XAxis, YAxis, Line } from "recharts";
    
    const Graph = ({ query }) => {
      const [data, setData] = useState([]);
    
      useEffect(() => {
        // API call to retrieve trend data
        axios.get(`/api/${query}`).then((res) => {
          setData(res.data);
        });
      }, [query]);
    
      return (
        <LineChart width={600} height={300} data={data}>
          <CartesianGrid strokeDasharray="3 3" />
          <XAxis dataKey="date" />
          <YAxis />
          <Line type="monotone" dataKey="value" stroke="#8884d8" />
        </LineChart>
      );
    };
    
    export default Graph;
    

    client/src/App.js

    import React, { useState } from "react";
    import Graph from "./components/graph";
    
    const App = () => {
      const [query, setQuery] = useState("");
    
      const handleSearch = () => {
        // API call to save trend data
        axios.post("/api/", { query });
      };
    
      return (
        <div>
          <input value={query} onChange={(e) => setQuery(e.target.value)} />
          <button onClick={handleSearch}>Search</button>
          <Graph query={query} />
        </div>
      );
    };
    
    export default App;
    

    client/src/index.js

    import React from "react";
    import ReactDOM from "react-dom";
    import App from "./App";
    
    ReactDOM.render(<App />, document.getElementById("root"));
    

    server/app.js

    const express = require("express");
    const cors = require("cors");
    const { connect } = require("./config/db");
    
    const app = express();
    app.use(cors());
    app.use(express.json());
    
    const scraperRoutes = require("./api/routes/scraper");
    app.use("/api", scraperRoutes);
    
    connect();
    
    app.listen(5000, () => console.log("Server started"));
    

    Note: The above code is not tested and may contain errors or require modifications based on the specific use case.