<!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"> <meta name="viewport"…
- User
h1 { color: #333; text-align: center; width: 100%; margin: 20px 0; } .table-container { width: 100%; max-width: 1200px; margin: 0 auto; background-color: #f2f2f2; } table { width: 100%; border-collapse: collapse; box-shadow: inset 1px 1px 2px #d1d1d1, inset -1px -1px 2px #ffffff; border-radius: 12px; overflow: hidden; } th, td { padding: 12px; text-align: left; overflow: hidden; white-space: nowrap; text-overflow: ellipsis; } th { background-color: #f2f2f2; box-shadow: inset 1px 1px 2px #d1d1d1, /* inset -1px -1px 2px #ffffff; */ /* color: #555; */ } th.name { width: 140px; } th.url { width: 100px; } th.link { width: 10px; } th.regex { width: 40px; } th.action { width: 100px; } td { background-color: #f2f2f2; color: #333; } input[type="text"] { width: calc(100% - 16px); padding: 8px; border: none; border-radius: 8px; box-shadow: inset 2px 2px 5px #bebebe, inset -2px -2px 5px #ffffff; background-color: #f2f2f2; } .debug-panel, .result-panel, .about-panel { position: fixed; top: 8%; left: 50%; transform: translateX(-50%); width: 80%; max-height: 80%; overflow-y: auto; background-color: #f2f2f2; box-shadow: 8px 8px 20px #bebebe, -8px -8px 20px #ffffff; border-radius: 10px; padding: 20px; display: none; z-index: 1000; } .close-btn { float: right; cursor: pointer; font-weight: bold; color: #333; } .overlay { position: fixed; top: 0; left: 0; width: 100%; height: 100%; background-color: rgba(0, 0, 0, 0.3); display: none; z-index: 999; } .auto-fetch-switch-container, .regex-toggle-container { display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px; width: 100%; max-width: 1200px; box-sizing: border-box; } .switch-label-container { display: flex; align-items: center; } .auto-fetch-switch-label, .regex-switch-label { margin-right: 10px; color: #333; font-weight: bold; } .auto-fetch-switch, .regex-switch { position: relative; display: inline-block; width: 50px; height: 25px; } .auto-fetch-switch input, .regex-switch input { display: none; } .slider { position: absolute; cursor: pointer; top: 0; left: 0; right: 0; bottom: 0; background-color: #ccc; transition: .4s; border-radius: 34px; } .slider:before { position: absolute; content: ""; height: 20px; width: 20px; left: 3px; bottom: 2px; background-color: white; transition: .4s; border-radius: 50%; } input:checked+.slider { background-color: #2196F3; } input:checked+.slider:before { transform: translateX(24px); } .request-access-link { color: #2196F3; font-weight: bold; text-decoration: none; } .import-export-container { position: fixed; top: 10px; right: 10px; display: flex; gap: 10px; z-index: 1001; } .about-container { position: fixed; top: 10px; left: 10px; display: flex; gap: 10px; z-index: 1001; } .hidden-input { display: none; } .icon-button { width: 36px; height: 36px; background-color: #f2f2f2; box-shadow: 5px 5px 10px #bebebe, -5px -5px 10px #ffffff; border: none; cursor: pointer; display: inline-flex; align-items: center; justify-content: center; border-radius: 50%; } .add-icon-button { width: 50px; height: 50px; border: none; background-color: transparent; } .icon-button img, .add-icon-button img { max-width: 70%; max-height: 70%; } .icon-button:hover { box-shadow: inset 2px 2px 5px #bebebe, inset -2px -2px 5px #ffffff; } .button-spacing { margin-right: 5px; } </style><div class="about-container"> <button class="icon-button" onclick="toggleAboutPanel()"> <img src="media/about.png" alt="About"> </button> </div> <h1>Open WebScraper v1.0</h1> <div class="table-container"> <div class="auto-fetch-switch-container"> <div class="switch-label-container"> <span class="auto-fetch-switch-label">Auto Fetch on Load</span> <label class="auto-fetch-switch"> <input type="checkbox" id="auto-fetch-switch" onchange="toggleAutoFetch(this.checked)"> <span class="slider"></span> </label> </div> <a href="https://cors-anywhere.herokuapp.com/corsdemo" class="request-access-link" target="_blank">[Not working? Request CORS-Anywhere]</a> </div> <table> <thead> <tr> <th class="name">Name</th> <th class="url">URL</th> <th class="link">🔗</th> <th class="selector">Selector/Regex</th> <th class="regex">Regex?</th> <th class="result">Results</th> <th class="actions"></th> </tr> </thead> <tbody id="scrape-table-body"> <tr> <td><input type="text" id="new-name" placeholder="Enter name"></td> <td><input type="text" id="new-url" placeholder="Enter URL"></td> <td></td> <td><input type="text" id="new-selector" placeholder="Enter Selector or Regex"></td> <td> <label class="regex-switch"> <input type="checkbox" id="new-regex"> <span class="slider"></span> </label> </td> <td colspan="2"><button class="add-icon-button" onclick="addNewRecord()"><img src="media/add.png" alt="Add"></button></td> </tr> </tbody> </table> </div> <div id="overlay" class="overlay" onclick="closePanels()"></div> <div id="debug-panel" class="debug-panel"> <span class="close-btn" onclick="toggleDebugPanel(null, true)">×</span> <pre id="debug-content"></pre> </div> <div id="result-panel" class="result-panel"> <span class="close-btn" onclick="toggleResultPanel(null, true)">×</span> <pre id="result-content"></pre> </div> <div id="about-panel" class="about-panel"> <span class="close-btn" onclick="toggleAboutPanel(true)">×</span> <!DOCTYPE html> <h1>WebScraper Pro</h1> <p> WebScraper Pro is an advanced web scraping tool designed to simplify data collection from websites with minimal setup. This tool is perfect for users who need to efficiently gather and process information from multiple web pages using both CSS selectors and regular expressions. </p> <h2>Key Features</h2> <ul> <li><strong>Automatic Cookie Management</strong>: WebScraper Pro simplifies session management by automatically handling cookies. Set up a list of websites, and the tool ensures smooth data retrieval on each page load.</li> <li><strong>Auto Reload Capability</strong>: No need to manually reload pages. The tool can automatically refresh pages when needed to keep your data up-to-date.</li> <li><strong>Flexible Data Extraction</strong>: Capture data using both CSS selectors and regular expressions. This dual support provides maximum flexibility and ease of use.</li> </ul> <h2>Use Cases</h2> <ul> <li><strong>Monitor Multiple Websites</strong>: Easily set up a list of websites to monitor and collect data automatically, saving you time and effort.</li> <li><strong>Content Aggregation</strong>: Use WebScraper Pro to gather specific information for research, content creation, or data analysis, leveraging its ability to parse content seamlessly.</li> </ul> <h2>Examples</h2> <p>WebScraper Pro allows you to use both selectors and regex to pull data from websites. Here are a few examples:</p> <h3>CSS Selector Examples</h3> <ul> <li>Extract the pre-content area from Wikipedia's "Hello" page: <pre><code>URL: https://en.wikipedia.org/wiki/Hello Selector: .vector-body-before-content Target the "Early uses" section: URL: https://en.wikipedia.org/wiki/Hello Selector: #Early_uses Capture elements with a title of "Aloha": URL: https://en.wikipedia.org/wiki/Hello Selector: [title="Aloha"]
<h3>Regular Expression Example</h3> <ul> <li>Extract price formats from the Wikipedia page: <pre><code>URL: https://en.wikipedia.org/wiki/Hello Regex: (?<=$)\d+.\d{2}
<h2>Getting Started</h2> <p> To get started with WebScraper Pro, clone the repository and follow the installation instructions in the <code>README.md</code>. Once set up, you can customize your scraping tasks by configuring the URLs and specifying the selectors or regex patterns you need. </p> <p> WebScraper Pro is the perfect choice for developers, analysts, and researchers looking to automate data collection and streamline their workflow. Contribute to this project by submitting pull requests and stay tuned for future updates! </p> <div>Icons Attribution:</div> <ul> <li>Multiply by Kozan from <a href="https://thenounproject.com/browse/icons/term/multiply/" target="_blank" title="multiply Icons">Noun Project</a> (CC BY 3.0) </li> <li>Multiply by Sunisih from <a href="https://thenounproject.com/browse/icons/term/multiply/" target="_blank" title="multiply Icons">Noun Project</a> (CC BY 3.0) </li> <li>Eye by Fahmi Ramdani from <a href="https://thenounproject.com/browse/icons/term/eye/" target="_blank" title="Eye Icons">Noun Project</a> (CC BY 3.0)</li> <li>add by Elin Erkani from <a href="https://thenounproject.com/browse/icons/term/add/" target="_blank" title="add Icons">Noun Project</a> (CC BY 3.0)</li> <li>clear by iconmas from <a href="https://thenounproject.com/browse/icons/term/clear/" target="_blank" title="clear Icons">Noun Project</a> (CC BY 3.0)</li> </ul> </div> <script src="https://cdn.jsdelivr.net/npm/axios/dist/axios.min.js"></script> <script> const recordsKey = 'scrapeRecords'; const autoFetchKey = 'autoFetch'; const maxResultLength = 50; document.addEventListener('DOMContentLoaded', () => { const autoFetchState = localStorage.getItem(autoFetchKey); const autoFetch = (autoFetchState === 'true'); const autoFetchSwitch = document.getElementById('auto-fetch-switch'); autoFetchSwitch.checked = autoFetch; loadRecords(autoFetch); }); function exportData() { const records = JSON.parse(localStorage.getItem(recordsKey)) || []; const blob = new Blob([JSON.stringify(records)], { type: 'application/json' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = 'scrape_records.json'; document.body.appendChild(a); a.click(); document.body.removeChild(a); } function importData(event) { const file = event.target.files[0]; if (!file) return; const reader = new FileReader(); reader.onload = function (evt) { const data = JSON.parse(evt.target.result); localStorage.setItem(recordsKey, JSON.stringify(data)); document.location.reload(); }; reader.readAsText(file); } function toggleAutoFetch(isEnabled) { localStorage.setItem(autoFetchKey, isEnabled); if (isEnabled) { executeAllRecords(); } } function loadRecords(autoFetch) { const records = JSON.parse(localStorage.getItem(recordsKey)) || []; records.forEach((record, index) => { addTableRow(record, index); if (autoFetch) { executeRecord(index); } }); } function addTableRow({ name, url, selector, isRegex, result, debugInfo }, index) { const tableBody = document.getElementById('scrape-table-body'); const row = document.createElement('tr'); row.innerHTML = ` <td><input type="text" value="${escapeHtml(name)}" onchange="updateRecord(${index}, 'name', this.value)" onkeydown="checkEnter(event, ${index})"></td> <td><input type="text" value="${escapeHtml(url)}" onchange="updateRecord(${index}, 'url', this.value)" onkeydown="checkEnter(event, ${index})"></td> <td><a class="request-access-link" href="${escapeHtml(url)}" target="_blank">🔗</a></td> <td><input type="text" value="${escapeHtml(selector)}" onchange="updateRecord(${index}, 'selector', this.value)" onkeydown="checkEnter(event, ${index})"></td> <td> <label class="regex-switch"> <input type="checkbox" id="regex-${index}" onchange="updateRecord(${index}, 'isRegex', this.checked)" ${isRegex ? 'checked' : ''}> <span class="slider"></span> </label> </td> <td id="result-${index}">${escapeHtml(result || '')}</td> <td> <button class="icon-button" onclick="executeRecord(${index})"> <img src="media/run.png" alt="Run"> </button> <button class="icon-button button-spacing" onclick="toggleDebugPanel(${index}, false)"> <img src="media/info.png" alt="Info"> </button> <button class="icon-button" onclick="removeRecord(${index})"> <img src="media/delete.png" alt="Delete"> </button> </td> `; tableBody.insertBefore(row, tableBody.lastElementChild); } function checkEnter(event, index) { if (event.key === 'Enter') { updateRecord(index, 'result', ''); executeRecord(index); } } function addNewRecord() { const name = document.getElementById('new-name').value.trim(); const url = document.getElementById('new-url').value.trim(); const selector = document.getElementById('new-selector').value.trim(); const isRegex = document.getElementById('new-regex').checked; if (!name || !url || !selector) { alert('All fields are required!'); return; } const record = { name, url, selector, isRegex, result: '', debugInfo: {} }; let records = JSON.parse(localStorage.getItem(recordsKey)) || []; records.push(record); localStorage.setItem(recordsKey, JSON.stringify(records)); const index = records.length - 1; addTableRow(record, index); executeRecord(index); document.getElementById('new-name').value = ''; document.getElementById('new-url').value = ''; document.getElementById('new-selector').value = ''; document.getElementById('new-regex').checked = false; } function executeRecord(index) { const records = JSON.parse(localStorage.getItem(recordsKey)); const record = records[index]; document.getElementById(`result-${index}`).textContent = '⏳'; executeScraping(record.url, record.selector, record.isRegex, index); } function executeScraping(url, selector, isRegex, index) { const proxy = 'https://cors-anywhere.herokuapp.com/'; const options = { headers: { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', 'Accept-Language': 'en-US,en;q=0.9' } }; axios.get(proxy + url, options) .then(response => { const fetchedData = response.data; let resultText = 'No matches'; if (isRegex) { try { const regex = new RegExp(selector); const match = regex.exec(fetchedData); resultText = match ? match[0] : 'No matches'; } catch (e) { console.error('Invalid regex:', e); resultText = 'Invalid regex'; } } else { const parser = new DOMParser(); const doc = parser.parseFromString(fetchedData, 'text/html'); const resultElement = doc.querySelector(selector); resultText = resultElement ? resultElement.textContent.trim() : 'No matches'; } document.getElementById(`result-${index}`).textContent = resultText.substring(0, maxResultLength) + (resultText.length > maxResultLength ? '...' : ''); const records = JSON.parse(localStorage.getItem(recordsKey)); records[index].result = resultText; records[index].debugInfo = { html: fetchedData, timestamp: new Date().toISOString() }; localStorage.setItem(recordsKey, JSON.stringify(records)); }) .catch(error => { console.error(`Error fetching the URL: ${url}`, error); document.getElementById(`result-${index}`).textContent = 'Error occurred'; const records = JSON.parse(localStorage.getItem(recordsKey)); records[index].result = 'Error occurred'; records[index].debugInfo = { error: error.toString(), timestamp: new Date().toISOString() }; localStorage.setItem(recordsKey, JSON.stringify(records)); }); } function updateRecord(index, field, value) { const records = JSON.parse(localStorage.getItem(recordsKey)); records[index][field] = value; localStorage.setItem(recordsKey, JSON.stringify(records)); } function removeRecord(index) { let records = JSON.parse(localStorage.getItem(recordsKey)); records.splice(index, 1); localStorage.setItem(recordsKey, JSON.stringify(records)); document.location.reload(); } function toggleDebugPanel(index, close = false) { const debugPanel = document.getElementById('debug-panel'); const debugContent = document.getElementById('debug-content'); const overlay = document.getElementById('overlay'); const records = JSON.parse(localStorage.getItem(recordsKey)) || []; if (close || index === null) { debugPanel.style.display = 'none'; overlay.style.display = 'none'; debugContent.innerHTML = ''; return; } const debugInfo = records[index].debugInfo; if (debugInfo) { debugContent.innerHTML = `<strong>Timestamp:</strong> ${debugInfo.timestamp}<br>HTML:{debugInfo.error || 'None'}`; } else { debugContent.innerHTML = 'No debug information available'; }
debugPanel.style.display = 'block'; overlay.style.display = 'block'; } function toggleAboutPanel(close = false) { const aboutPanel = document.getElementById('about-panel'); const overlay = document.getElementById('overlay'); if (close) { aboutPanel.style.display = 'none'; overlay.style.display = 'none'; return; } aboutPanel.style.display = 'block'; overlay.style.display = 'block'; } function closePanels() { toggleDebugPanel(null, true); toggleAboutPanel(true); } function escapeHtml(text) { if (typeof text !== "string") return ''; const map = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }; return text.replace(/[&<>"']/g, function(m) { return map[m]; }); } function executeAllRecords() { const records = JSON.parse(localStorage.getItem(recordsKey)) || []; records.forEach((_, index) => { executeRecord(index); }); } function clearAllRecords() { if (confirm('Are you sure you want to clear all records? This action cannot be undone.')) { localStorage.removeItem(recordsKey); document.getElementById('scrape-table-body').innerHTML = ` <tr> <td><input type="text" id="new-name" placeholder="Enter name"></td> <td><input type="text" id="new-url" placeholder="Enter URL"></td> <td></td> <td><input type="text" id="new-selector" placeholder="Enter Selector or Regex"></td> <td> <label class="regex-switch"> <input type="checkbox" id="new-regex"> <span class="slider"></span> </label> </td> <td colspan="2"><button class="add-icon-button" onclick="addNewRecord()"><img src="media/add.png" alt="Add"></button></td> </tr> `; } } </script>currently this code is capturing user input and preference and saving it to localstorage and allow output to file and import from file. But it is also saving the result along with the full debug html information. We don't want the full html debug info of local storage and when exporting as it is a waste of space. We want to the user input preference to be saved and export/import, while the debug information is still show upon pressing the info button when it is available during that session
show the code and only, provide the full code don't omit anything
- Assistant
<!DOCTYPE html> <html lang="en"> <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <title>Open WebScraper 1.0</title> <style> /* Basic styling for the page and elements */ body { font-family: Arial, sans-serif; margin: 0; padding: 0; background-color: #f2f2f2; display: flex; justify-content: center; align-items: flex-start; flex-direction: column; min-height: 100vh; width: 100%; box-sizing: border-box; } h1 { color: #333; text-align: center; width: 100%; margin: 20px 0; } .table-container { width: 100%; max-width: 1200px; margin: 0 auto; background-color: #f2f2f2; } table { width: 100%; border-collapse: collapse; box-shadow: inset 1px 1px 2px #d1d1d1, inset -1px -1px 2px #ffffff; border-radius: 12px; overflow: hidden; } th, td { padding: 12px; text-align: left; overflow: hidden; white-space: nowrap; text-overflow: ellipsis; } th { background-color: #f2f2f2; box-shadow: inset 1px 1px 2px #d1d1d1; } th.name { width: 140px; } th.url { width: 100px; } th.link { width: 10px; } th.selector { width: 40px; } th.regex { width: 40px; } th.result { width: 100px; } th.actions { width: 100px; } td { background-color: #f2f2f2; color: #333; } input[type="text"] { width: calc(100% - 16px); padding: 8px; border: none; border-radius: 8px; box-shadow: inset 2px 2px 5px #bebebe, inset -2px -2px 5px #ffffff; background-color: #f2f2f2; } .debug-panel, .result-panel, .about-panel { position: fixed; top: 8%; left: 50%; transform: translateX(-50%); width: 80%; max-height: 80%; overflow-y: auto; background-color: #f2f2f2; box-shadow: 8px 8px 20px #bebebe, -8px -8px 20px #ffffff; border-radius: 10px; padding: 20px; display: none; z-index: 1000; } .close-btn { float: right; cursor: pointer; font-weight: bold; color: #333; } .overlay { position: fixed; top: 0; left: 0; width: 100%; height: 100%; background-color: rgba(0, 0, 0, 0.3); display: none; z-index: 999; } .auto-fetch-switch-container, .regex-toggle-container { display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px; width: 100%; max-width: 1200px; box-sizing: border-box; } .switch-label-container { display: flex; align-items: center; } .auto-fetch-switch-label, .regex-switch-label { margin-right: 10px; color: #333; font-weight: bold; } .auto-fetch-switch, .regex-switch { position: relative; display: inline-block; width: 50px; height: 25px; } .auto-fetch-switch input, .regex-switch input { display: none; } .slider { position: absolute; cursor: pointer; top: 0; left: 0; right: 0; bottom: 0; background-color: #ccc; transition: .4s; border-radius: 34px; } .slider:before { position: absolute; content: ""; height: 20px; width: 20px; left: 3px; bottom: 2px; background-color: white; transition: .4s; border-radius: 50%; } input:checked+.slider { background-color: #2196F3; } input:checked+.slider:before { transform: translateX(24px); } .request-access-link { color: #2196F3; font-weight: bold; text-decoration: none; } .import-export-container { position: fixed; top: 10px; right: 10px; display: flex; gap: 10px; z-index: 1001; } .about-container { position: fixed; top: 10px; left: 10px; display: flex; gap: 10px; z-index: 1001; } .hidden-input { display: none; } .icon-button { width: 36px; height: 36px; background-color: #f2f2f2; box-shadow: 5px 5px 10px #bebebe, -5px -5px 10px #ffffff; border: none; cursor: pointer; display: inline-flex; align-items: center; justify-content: center; border-radius: 50%; } .add-icon-button { width: 50px; height: 50px; border: none; background-color: transparent; } .icon-button img, .add-icon-button img { max-width: 70%; max-height: 70%; } .icon-button:hover { box-shadow: inset 2px 2px 5px #bebebe, inset -2px -2px 5px #ffffff; } .button-spacing { margin-right: 5px; } </style> </head> <body> <div class="import-export-container"> <button class="icon-button" onclick="exportData()"> <img src="media/export.png" alt="Export"> </button> <button class="icon-button" onclick="document.getElementById('import-file').click()"> <img src="media/import.png" alt="Import"> </button> <button class="icon-button" onclick="toggleAboutPanel()"> <img src="media/about.png" alt="About"> </button> <button class="icon-button" onclick="clearAllRecords()"> <img src="media/clear.png" alt="Clear"> </button> <input type="file" id="import-file" class="hidden-input" accept=".json" onchange="importData(event)"> </div> <div class="about-container"> <button class="icon-button" onclick="toggleAboutPanel()"> <img src="media/about.png" alt="About"> </button> </div> <h1>Open WebScraper v1.0</h1> <div class="table-container"> <div class="auto-fetch-switch-container"> <div class="switch-label-container"> <span class="auto-fetch-switch-label">Auto Fetch on Load</span> <label class="auto-fetch-switch"> <input type="checkbox" id="auto-fetch-switch" onchange="toggleAutoFetch(this.checked)"> <span class="slider"></span> </label> </div> <a href="https://cors-anywhere.herokuapp.com/corsdemo" class="request-access-link" target="_blank">[Not working? Request CORS-Anywhere]</a> </div> <table> <thead> <tr> <th class="name">Name</th> <th class="url">URL</th> <th class="link">🔗</th> <th class="selector">Selector/Regex</th> <th class="regex">Regex?</th> <th class="result">Results</th> <th class="actions"></th> </tr> </thead> <tbody id="scrape-table-body"> <tr> <td><input type="text" id="new-name" placeholder="Enter name"></td> <td><input type="text" id="new-url" placeholder="Enter URL"></td> <td></td> <td><input type="text" id="new-selector" placeholder="Enter Selector or Regex"></td> <td> <label class="regex-switch"> <input type="checkbox" id="new-regex"> <span class="slider"></span> </label> </td> <td colspan="2"><button class="add-icon-button" onclick="addNewRecord()"><img src="media/add.png" alt="Add"></button></td> </tr> </tbody> </table> </div> <div id="overlay" class="overlay" onclick="closePanels()"></div> <div id="debug-panel" class="debug-panel"> <span class="close-btn" onclick="toggleDebugPanel(null, true)">×</span> <pre id="debug-content"></pre> </div> <div id="result-panel" class="result-panel"> <span class="close-btn" onclick="toggleResultPanel(null, true)">×</span> <pre id="result-content"></pre> </div> <div id="about-panel" class="about-panel"> <span class="close-btn" onclick="toggleAboutPanel(true)">×</span> <h1>WebScraper Pro</h1> <p> WebScraper Pro is an advanced web scraping tool designed to simplify data collection from websites with minimal setup. This tool is perfect for users who need to efficiently gather and process information from multiple web pages using both CSS selectors and regular expressions. </p> <h2>Key Features</h2> <ul> <li><strong>Automatic Cookie Management</strong>: WebScraper Pro simplifies session management by automatically handling cookies. Set up a list of websites, and the tool ensures smooth data retrieval on each page load.</li> <li><strong>Auto Reload Capability</strong>: No need to manually reload pages. The tool can automatically refresh pages when needed to keep your data up-to-date.</li> <li><strong>Flexible Data Extraction</strong>: Capture data using both CSS selectors and regular expressions. This dual support provides maximum flexibility and ease of use.</li> </ul> <h2>Use Cases</h2> <ul> <li><strong>Monitor Multiple Websites</strong>: Easily set up a list of websites to monitor and collect data automatically, saving you time and effort.</li> <li><strong>Content Aggregation</strong>: Use WebScraper Pro to gather specific information for research, content creation, or data analysis, leveraging its ability to parse content seamlessly.</li> </ul> <h2>Examples</h2> <p>WebScraper Pro allows you to use both selectors and regex to pull data from websites. Here are a few examples:</p> <h3>CSS Selector Examples</h3> <ul> <li>Extract the pre-content area from Wikipedia's "Hello" page: <pre><code> URL: https://en.wikipedia.org/wiki/Hello Selector: .vector-body-before-content </code></pre> </li> <li>Target the "Early uses" section: <pre><code> URL: https://en.wikipedia.org/wiki/Hello Selector: #Early_uses </code></pre> </li> <li>Capture elements with a title of "Aloha": <pre><code> URL: https://en.wikipedia.org/wiki/Hello Selector: [title="Aloha"] </code></pre> </li> </ul> <h3>Regular Expression Example</h3> <ul> <li>Extract price formats from the Wikipedia page: <pre><code> URL: https://en.wikipedia.org/wiki/Hello Regex: (?<=\$)\d+\.\d{2} </code></pre> </li> </ul> <h2>Getting Started</h2> <p> To get started with WebScraper Pro, clone the repository and follow the installation instructions in the <code>README.md</code>. Once set up, you can customize your scraping tasks by configuring the URLs and specifying the selectors or regex patterns you need. </p> <p> WebScraper Pro is the perfect choice for developers, analysts, and researchers looking to automate data collection and streamline their workflow. Contribute to this project by submitting pull requests and stay tuned for future updates! </p> <div>Icons Attribution:</div> <ul> <li>Multiply by Kozan from <a href="https://thenounproject.com/browse/icons/term/multiply/" target="_blank" title="multiply Icons">Noun Project</a> (CC BY 3.0) </li> <li>Multiply by Sunisih from <a href="https://thenounproject.com/browse/icons/term/multiply/" target="_blank" title="multiply Icons">Noun Project</a> (CC BY 3.0) </li> <li>Eye by Fahmi Ramdani from <a href="https://thenounproject.com/browse/icons/term/eye/" target="_blank" title="Eye Icons">Noun Project</a> (CC BY 3.0)</li> <li>Add by Elin Erkani from <a href="https://thenounproject.com/browse/icons/term/add/" target="_blank" title="add Icons">Noun Project</a> (CC BY 3.0)</li> <li>Clear by iconmas from <a href="https://thenounproject.com/browse/icons/term/clear/" target="_blank" title="clear Icons">Noun Project</a> (CC BY 3.0)</li> </ul> </div> <script src="https://cdn.jsdelivr.net/npm/axios/dist/axios.min.js"></script> <script> const recordsKey = 'scrapeRecords'; const autoFetchKey = 'autoFetch'; const maxResultLength = 50; document.addEventListener('DOMContentLoaded', () => { const autoFetchState = localStorage.getItem(autoFetchKey); const autoFetch = (autoFetchState === 'true'); const autoFetchSwitch = document.getElementById('auto-fetch-switch'); autoFetchSwitch.checked = autoFetch; loadRecords(autoFetch); }); function exportData() { const records = JSON.parse(localStorage.getItem(recordsKey)) || []; const exportRecords = records.map(({ name, url, selector, isRegex, result }) => ({ name, url, selector, isRegex, result })); const blob = new Blob([JSON.stringify(exportRecords)], { type: 'application/json' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = 'scrape_records.json'; document.body.appendChild(a); a.click(); document.body.removeChild(a); } function importData(event) { const file = event.target.files[0]; if (!file) return; const reader = new FileReader(); reader.onload = function (evt) { const data = JSON.parse(evt.target.result); localStorage.setItem(recordsKey, JSON.stringify(data)); document.location.reload(); }; reader.readAsText(file); } function toggleAutoFetch(isEnabled) { localStorage.setItem(autoFetchKey, isEnabled); if (isEnabled) { executeAllRecords(); } } function loadRecords(autoFetch) { const records = JSON.parse(localStorage.getItem(recordsKey)) || []; records.forEach((record, index) => { addTableRow(record, index); if (autoFetch) { executeRecord(index); } }); } function addTableRow({ name, url, selector, isRegex, result }, index) { const tableBody = document.getElementById('scrape-table-body'); const row = document.createElement('tr'); row.innerHTML = ` <td><input type="text" value="${escapeHtml(name)}" onchange="updateRecord(${index}, 'name', this.value)" onkeydown="checkEnter(event, ${index})"></td> <td><input type="text" value="${escapeHtml(url)}" onchange="updateRecord(${index}, 'url', this.value)" onkeydown="checkEnter(event, ${index})"></td> <td><a class="request-access-link" href="${escapeHtml(url)}" target="_blank">🔗</a></td> <td><input type="text" value="${escapeHtml(selector)}" onchange="updateRecord(${index}, 'selector', this.value)" onkeydown="checkEnter(event, ${index})"></td> <td> <label class="regex-switch"> <input type="checkbox" id="regex-${index}" onchange="updateRecord(${index}, 'isRegex', this.checked)" ${isRegex ? 'checked' : ''}> <span class="slider"></span> </label> </td> <td id="result-${index}">${escapeHtml(result || '')}</td> <td> <button class="icon-button" onclick="executeRecord(${index})"> <img src="media/run.png" alt="Run"> </button> <button class="icon-button button-spacing" onclick="toggleDebugPanel(${index}, false)"> <img src="media/info.png" alt="Info"> </button> <button class="icon-button" onclick="removeRecord(${index})"> <img src="media/delete.png" alt="Delete"> </button> </td> `; tableBody.insertBefore(row, tableBody.lastElementChild); } function checkEnter(event, index) { if (event.key === 'Enter') { updateRecord(index, 'result', ''); executeRecord(index); } } function addNewRecord() { const name = document.getElementById('new-name').value.trim(); const url = document.getElementById('new-url').value.trim(); const selector = document.getElementById('new-selector').value.trim(); const isRegex = document.getElementById('new-regex').checked; if (!name || !url || !selector) { alert('All fields are required!'); return; } const record = { name, url, selector, isRegex, result: '' }; let records = JSON.parse(localStorage.getItem(recordsKey)) || []; records.push(record); localStorage.setItem(recordsKey, JSON.stringify(records)); const index = records.length - 1; addTableRow(record, index); executeRecord(index); document.getElementById('new-name').value = ''; document.getElementById('new-url').value = ''; document.getElementById('new-selector').value = ''; document.getElementById('new-regex').checked = false; } function executeRecord(index) { const records = JSON.parse(localStorage.getItem(recordsKey)); const record = records[index]; document.getElementById(`result-${index}`).textContent = '⏳'; executeScraping(record.url, record.selector, record.isRegex, index); } function executeScraping(url, selector, isRegex, index) { const proxy = 'https://cors-anywhere.herokuapp.com/'; const options = { headers: { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', 'Accept-Language': 'en-US,en;q=0.9' } }; axios.get(proxy + url, options) .then(response => { const fetchedData = response.data; let resultText = 'No matches'; if (isRegex) { try { const regex = new RegExp(selector); const match = regex.exec(fetchedData); resultText = match ? match[0] : 'No matches'; } catch (e) { console.error('Invalid regex:', e); resultText = 'Invalid regex'; } } else { const parser = new DOMParser(); const doc = parser.parseFromString(fetchedData, 'text/html'); const resultElement = doc.querySelector(selector); resultText = resultElement ? resultElement.textContent.trim() : 'No matches'; } document.getElementById(`result-${index}`).textContent = resultText.substring(0, maxResultLength) + (resultText.length > maxResultLength ? '...' : ''); const records = JSON.parse(localStorage.getItem(recordsKey)); records[index].result = resultText; localStorage.setItem(recordsKey, JSON.stringify(records)); }) .catch(error => { console.error(`Error fetching the URL: ${url}`, error); document.getElementById(`result-${index}`).textContent = 'Error occurred'; const records = JSON.parse(localStorage.getItem(recordsKey)); records[index].result = 'Error occurred'; localStorage.setItem(recordsKey, JSON.stringify(records)); }); } function updateRecord(index, field, value) { const records = JSON.parse(localStorage.getItem(recordsKey)); records[index][field] = value; localStorage.setItem(recordsKey, JSON.stringify(records)); } function removeRecord(index) { let records = JSON.parse(localStorage.getItem(recordsKey)); records.splice(index, 1); localStorage.setItem(recordsKey, JSON.stringify(records)); document.location.reload(); } function toggleDebugPanel(index, close = false) { const debugPanel = document.getElementById('debug-panel'); const debugContent = document.getElementById('debug-content'); const overlay = document.getElementById('overlay'); const records = JSON.parse(localStorage.getItem(recordsKey)) || []; if (close || index === null) { debugPanel.style.display = 'none'; overlay.style.display = 'none'; debugContent.innerHTML = ''; return; } const debugInfo = records[index].debugInfo; if (debugInfo) { debugContent.innerHTML = `<strong>Timestamp:</strong> ${debugInfo.timestamp}<br> <strong>Error:</strong> ${debugInfo.error || 'None'}`; } else { debugContent.innerHTML = 'No debug information available'; } debugPanel.style.display = 'block'; overlay.style.display = 'block'; } function toggleAboutPanel(close = false) { const aboutPanel = document.getElementById('about-panel'); const overlay = document.getElementById('overlay'); if (close) { aboutPanel.style.display = 'none'; overlay.style.display = 'none'; return; } aboutPanel.style.display = 'block'; overlay.style.display = 'block'; } function closePanels() { toggleDebugPanel(null, true); toggleAboutPanel(true); } function escapeHtml(text) { if (typeof text !== "string") return ''; const map = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }; return text.replace(/[&<>"']/g, function (m) { return map[m]; }); } function executeAllRecords() { const records = JSON.parse(localStorage.getItem(recordsKey)) || []; records.forEach((_, index) => { executeRecord(index); }); } function clearAllRecords() { if (confirm('Are you sure you want to clear all records? This action cannot be undone.')) { localStorage.removeItem(recordsKey); document.getElementById('scrape-table-body').innerHTML = ` <tr> <td><input type="text" id="new-name" placeholder="Enter name"></td> <td><input type="text" id="new-url" placeholder="Enter URL"></td> <td></td> <td><input type="text" id="new-selector" placeholder="Enter Selector or Regex"></td> <td> <label class="regex-switch"> <input type="checkbox" id="new-regex"> <span class="slider"></span> </label> </td> <td colspan="2"><button class="add-icon-button" onclick="addNewRecord()"><img src="media/add.png" alt="Add"></button></td> </tr> `; } } </script> </body> </html>