gpt-3.5-turbo-0301· OpenAI WildChat-4.8M (AI2)
Act as: MERN Developer Technology stack: Node.js, Express.js, MongoDB, Mongoose, ReactJS…
- User
Act as: MERN Developer
Technology stack: Node.js, Express.js, MongoDB, Mongoose, ReactJS
Functionality: scrape google trends and populate graph in ui
Task: Make an API that scrape google trends data and populate graph in UI. Generate only code. Mention file names with folder structure.
- Assistant
Folder Structure:
- server
- api
- controllers
- scraper.js
- routes
- scraper.js
- controllers
- config
- db.js
- models
- trend.js
- utils
- scraper.js
- app.js
- api
- client
- public
- index.html
- index.css
- src
- components
- graph.jsx
- App.js
- index.js
- components
- public
File names and Code:
server/api/controllers/scraper.js
const Trend = require("../models/trend"); const scraper = require("../utils/scraper"); // Function to scrape and save trend data const saveTrendData = async (query) => { try { const data = await scraper.scrape(query); // Save trend data to database const trend = new Trend({ query, data }); await trend.save(); return true; } catch (error) { console.log(error.message); return false; } }; // Controller to handle saving trend data exports.saveTrend = async (req, res) => { const { query } = req.body; const saved = await saveTrendData(query); if (saved) { res.send("Trend data saved successfully!"); } else { res.status(500).send("Error occurred while saving data"); } }; // Function to retrieve trend data exports.getTrendData = async (req, res) => { const { query } = req.params; try { // Retrieve trend data from database const trend = await Trend.findOne({ query }); res.send(trend); } catch (error) { res.status(500).send("Error occurred while retrieving data"); } };server/api/routes/scraper.js
const express = require("express"); const scraperController = require("../controllers/scraper"); const router = express.Router(); // Route to save trend data to database router.post("/", scraperController.saveTrend); // Route to get trend data from database router.get("/:query", scraperController.getTrendData); module.exports = router;server/config/db.js
const mongoose = require("mongoose"); // Database connection function const connect = async () => { try { await mongoose.connect(MONGODB_URI, { useNewUrlParser: true, useUnifiedTopology: true, }); console.log("Database connected successfully"); } catch (error) { console.log(error.message); } }; module.exports = { connect };server/models/trend.js
const mongoose = require("mongoose"); // Schema for trend data const TrendSchema = new mongoose.Schema({ query: { type: String, required: true }, data: { type: Object, required: true }, }); const Trend = mongoose.model("Trend", TrendSchema); module.exports = Trend;server/utils/scraper.js
const puppeteer = require("puppeteer"); // Function to scrape trend data exports.scrape = async (query) => { const browser = await puppeteer.launch(); const page = await browser.newPage(); await page.goto(`https://trends.google.com/trends/explore?q=${query}`); await page.waitForSelector(".widget-div.widget-gpt"); const data = await page.evaluate(() => { const tableRows = Array.from( document.querySelectorAll(".widget-div.widget-gpt tbody tr") ); return tableRows.map((row) => { const cells = row.querySelectorAll("td"); return { date: cells[0].textContent.trim(), value: cells[1].textContent.trim(), }; }); }); await browser.close(); return data; };client/src/components/graph.jsx
import React, { useState, useEffect } from "react"; import axios from "axios"; import { LineChart, CartesianGrid, XAxis, YAxis, Line } from "recharts"; const Graph = ({ query }) => { const [data, setData] = useState([]); useEffect(() => { // API call to retrieve trend data axios.get(`/api/${query}`).then((res) => { setData(res.data); }); }, [query]); return ( <LineChart width={600} height={300} data={data}> <CartesianGrid strokeDasharray="3 3" /> <XAxis dataKey="date" /> <YAxis /> <Line type="monotone" dataKey="value" stroke="#8884d8" /> </LineChart> ); }; export default Graph;client/src/App.js
import React, { useState } from "react"; import Graph from "./components/graph"; const App = () => { const [query, setQuery] = useState(""); const handleSearch = () => { // API call to save trend data axios.post("/api/", { query }); }; return ( <div> <input value={query} onChange={(e) => setQuery(e.target.value)} /> <button onClick={handleSearch}>Search</button> <Graph query={query} /> </div> ); }; export default App;client/src/index.js
import React from "react"; import ReactDOM from "react-dom"; import App from "./App"; ReactDOM.render(<App />, document.getElementById("root"));server/app.js
const express = require("express"); const cors = require("cors"); const { connect } = require("./config/db"); const app = express(); app.use(cors()); app.use(express.json()); const scraperRoutes = require("./api/routes/scraper"); app.use("/api", scraperRoutes); connect(); app.listen(5000, () => console.log("Server started"));Note: The above code is not tested and may contain errors or require modifications based on the specific use case.
- server