// ==UserScript== // @name Advanced Web Crawler Enhanced Pro Optimized v6.2.1 //…
- User
// ==UserScript== // @name Advanced Web Crawler Enhanced Pro Optimized v6.2.1 // @namespace http://tampermonkey.net/ // @version 6.2.1 // @description Optimized and enhanced version of Advanced Web Crawler Enhanced Pro with improved error handling, asynchronous processing, performance optimizations, enhanced user experience, responsive design, accessibility features, expanded search engine support, and enhanced security measures. // @author Your Name // @match https://www.google.com/search* // @match https://www.bing.com/search* // @match https://search.yahoo.com/search* // @match https://duckduckgo.com/* // @match https://www.baidu.com/s* // @match https://www.yandex.com/search/* // @match https://search.naver.com/search* // @match https://search.daum.net/search* // @match https://www.ask.com/* // @match https://www.ecosia.org/search* // @match https://www.startpage.com/* // @match https://searx.me/* // @match https://www.dogpile.com/* // @match https://www.bing.com/images* // @grant GM_xmlhttpRequest // @grant GM_setValue // @grant GM_getValue // @grant GM_deleteValue // @grant GM_addStyle // @connect google.com // @connect bing.com // @connect yahoo.com // @connect duckduckgo.com // @connect baidu.com // @connect yandex.com // @connect naver.com // @connect daum.net // @connect ask.com // @connect ecosia.org // @connect startpage.com // @connect searx.me // @connect dogpile.com // @require https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js // @require https://cdn.jsdelivr.net/npm/chart.js // @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js // @require https://cdnjs.cloudflare.com/ajax/libs/idb/7.0.1/idb.min.js // ==/UserScript==
(function() { 'use strict';
// Configuration Module const Config = (() => { const DEFAULT_SETTINGS = { delay: 2000, delayRandom: 500, maxPages: 3, userAgents: [ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)', 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15', 'Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15', 'AdvancedWebCrawler/6.2 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' ], userAgentIndex: 0, blacklist: [], whitelist: [], contentTypes: { include: ['text/html'], exclude: ['application/pdf', 'image/jpeg', 'image/png', 'video/mp4'] }, timeout: 8000, includeKeywords: [], excludeKeywords: [], regexInclude: [], regexExclude: [], language: [], minWordCount: 100, maxWordCount: 10000, maxDepth: 2, proxy: '', dataFields: { name: true, url: true, description: true, html: true, category: true, quality: true, language: true, wordCount: true, metaTags: false, headings: false }, schedule: { enabled: false, startTime: '00:00', interval: 3600000 }, searchEngines: { google: true, bing: true, yahoo: true, duckduckgo: true, baidu: true, yandex: true, naver: true, daum: true, ask: true, ecosia: true, startpage: true, searx: true, dogpile: true, bingimages: true }, retryAttempts: 3, retryDelay: 2000, respectRobots: true, maxConcurrentRequests: 3, dataVisualization: true, exportFormats: { csv: true, json: true, xml: true, xls: true }, advancedLogging: true, enableCaptchas: false, customKeywords: {}, currentTab: 'logs', isCrawling: false, progress: 0, currentPage: 1, currentDepth: 0 }; const settings = { ...DEFAULT_SETTINGS, ...getSavedSettings() }; function getSavedSettings() { return GM_getValue('settings', DEFAULT_SETTINGS); } function saveSettings(newSettings) { GM_setValue('settings', newSettings); Object.assign(settings, newSettings); } function resetSettings() { GM_setValue('settings', DEFAULT_SETTINGS); Object.assign(settings, DEFAULT_SETTINGS); } return { DEFAULT_SETTINGS, settings, saveSettings, resetSettings }; })(); // IndexedDB Setup const DB = (() => { const dbPromise = idb.openDB('crawler-db', 1, { upgrade(db) { if (!db.objectStoreNames.contains('data')) { const store = db.createObjectStore('data', { keyPath: 'url' }); store.createIndex('category', 'category'); } if (!db.objectStoreNames.contains('logs')) { const store = db.createObjectStore('logs', { autoIncrement: true }); store.createIndex('type', 'type'); } if (!db.objectStoreNames.contains('visited')) { db.createObjectStore('visited', { keyPath: 'url' }); } if (!db.objectStoreNames.contains('previousCrawls')) { db.createObjectStore('previousCrawls', { autoIncrement: true }); } } }); async function addData(entry) { const db = await dbPromise; await db.put('data', entry); } async function getAllData() { const db = await dbPromise; return await db.getAll('data'); } async function clearData() { const db = await dbPromise; await db.clear('data'); } async function addLog(log) { const db = await dbPromise; await db.add('logs', log); maintainLogSize(); } async function getAllLogs() { const db = await dbPromise; return await db.getAll('logs'); } async function clearLogs() { const db = await dbPromise; await db.clear('logs'); } async function addVisited(url) { const db = await dbPromise; await db.put('visited', { url }); } async function isVisited(url) { const db = await dbPromise; const result = await db.get('visited', url); return !!result; } async function clearVisited() { const db = await dbPromise; await db.clear('visited'); } async function addPreviousCrawl(crawl) { const db = await dbPromise; await db.add('previousCrawls', crawl); } async function getPreviousCrawls() { const db = await dbPromise; return await db.getAll('previousCrawls'); } async function clearPreviousCrawls() { const db = await dbPromise; await db.clear('previousCrawls'); } async function maintainLogSize() { const db = await dbPromise; const count = await db.count('logs'); const maxLogs = 1000; if (count > maxLogs) { const keys = await db.getAllKeys('logs', IDBKeyRange.bound(0, maxLogs - 1)); await db.delete('logs', keys[0]); } } return { addData, getAllData, clearData, addLog, getAllLogs, clearLogs, addVisited, isVisited, clearVisited, addPreviousCrawl, getPreviousCrawls, clearPreviousCrawls }; })(); // Logger Module const Logger = (() => { async function log(message, type = 'info') { const timestamp = new Date().toLocaleString(); const logEntry = { timestamp, message, type }; await DB.addLog(logEntry); UI.updateLogs(); if (Config.settings.advancedLogging) { const styles = { 'info': 'color: blue;', 'warning': 'color: orange;', 'error': 'color: red;', 'success': 'color: green; font-weight: bold;' }; console.log(`%c[${timestamp}] [${type.toUpperCase()}] ${message}`, styles[type] || ''); } } async function getLogs() { return await DB.getAllLogs(); } return { log, getLogs }; })(); // Utility Module const Utils = (() => { function escape(str, type = 'html') { const map = { html: { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }, csv: { '"': '""' }, xml: { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }, }; return String(str).replace(new RegExp(`[${Object.keys(map[type]).join('')}]`, 'g'), match => map[type][match]); } function capitalize(str) { return str.charAt(0).toUpperCase() + str.slice(1); } function delay(ms) { return new Promise(res => setTimeout(res, ms)); } function generateColors(count) { return Array.from({length: count}, (_, i) => `hsl(${i * (360 / count)}, 70%, 50%)`); } function parseCustomKeywords(text) { const lines = text.split('\n').map(l => l.trim()).filter(l => l); const ck = {}; lines.forEach(line => { const [cat, kws] = line.split(':'); if (cat && kws) ck[cat.trim()] = kws.split(',').map(kw => kw.trim()).filter(kw => kw); }); return ck; } function formatCustomKeywords(ck) { return Object.entries(ck).map(([c, kws]) => `${c}:${kws.join(',')}`).join('\n'); } return { escape, capitalize, delay, generateColors, parseCustomKeywords, formatCustomKeywords }; })(); // UI Module const UI = (() => { function createUI() { GM_addStyle(` #crawler-container { position: fixed; top: 20px; right: 20px; width: 90%; max-width: 500px; height: 80%; background: #fff; border: 1px solid #ccc; z-index: 10000; font-family: Arial, sans-serif; box-shadow: 0 4px 8px rgba(0,0,0,0.2); overflow: hidden; resize: both; border-radius: 8px; min-width: 300px; min-height: 300px; } #crawler-header { display: flex; justify-content: space-between; align-items: center; background: #f1f1f1; border-bottom: 1px solid #ccc; padding: 10px; cursor: move; } #crawler-minimize { background: none; border: none; font-size: 18px; cursor: pointer; } #crawler-tabs { display: flex; background: #f1f1f1; border-bottom: 1px solid #ccc; width: 100%; } .crawler-tab { flex: 1; padding: 10px; text-align: center; cursor: pointer; font-weight: bold; } .crawler-tab.active { background: #ddd; } #crawler-content { padding: 10px; overflow: auto; height: calc(100% - 50px); } table { width: 100%; border-collapse: collapse; } th, td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; } th { background-color: #f2f2f2; cursor: pointer; } tr:hover { background-color: #f5f5f5; } .progress-bar { width: 100%; background-color: #f3f3f3; border: 1px solid #ccc; border-radius: 5px; overflow: hidden; margin-bottom: 10px; } .progress-bar-fill { height: 20px; background-color: #4caf50; width: 0%; transition: width 0.2s; } .modal { display: none; position: fixed; z-index: 10001; left: 0; top: 0; width: 100%; height: 100%; overflow: auto; background-color: rgba(0,0,0,0.4); } .modal-content { background-color: #fff; margin: 5% auto; padding: 20px; border: 1px solid #888; width: 90%; max-width: 800px; max-height: 80%; overflow-y: auto; border-radius: 8px; } .close { color: #aaa; float: right; font-size: 28px; font-weight: bold; cursor: pointer; } .control-buttons { margin: 10px 0; display: flex; flex-wrap: wrap; gap: 5px; } .control-buttons button { padding: 8px 12px; cursor: pointer; border: none; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px; } .control-buttons button:disabled { background-color: #a5d6a7; cursor: not-allowed; } .status-text { margin-top: 10px; font-weight: bold; font-size: 14px; } .settings-section { margin-bottom: 15px; } .settings-section label { display: block; margin-bottom: 5px; font-weight: bold; } .settings-section input[type="text"], .settings-section input[type="number"], .settings-section textarea, .settings-section select { width: 100%; padding: 5px; box-sizing: border-box; border: 1px solid #ccc; border-radius: 4px; font-size: 14px; } .settings-section .checkbox-group { display: flex; flex-wrap: wrap; gap: 10px; } .settings-section .checkbox-group label { font-size: 14px; } .tooltip { position: relative; display: inline-block; cursor: help; font-size: 14px; } .tooltip .tooltiptext { visibility: hidden; width: 220px; background-color: #555; color: #fff; text-align: center; border-radius: 6px; padding: 5px; position: absolute; z-index: 1; bottom: 125%; left: 50%; margin-left: -110px; opacity: 0; transition: opacity 0.3s; } .tooltip:hover .tooltiptext { visibility: visible; opacity: 1; } #crawler-container::-webkit-resizer { background: #f1f1f1; border: none; } #data-visualization { margin-top: 20px; } #data-chart { width: 100%; height: 300px; } #modal-html-content { white-space: pre-wrap; word-wrap: break-word; background-color: #f9f9f9; padding: 10px; border-radius: 4px; } #crawler-tabs .crawler-tab:last-child { flex: 1; } #previous-crawls { margin-top: 20px; } #previous-crawls table { width: 100%; } #previous-crawls th, #previous-crawls td { padding: 5px; } #reset-data { background-color: #f44336; } .spinner { border: 4px solid #f3f3f3; border-top: 4px solid #3498db; border-radius: 50%; width: 24px; height: 24px; animation: spin 2s linear infinite; display: inline-block; vertical-align: middle; } @keyframes spin { 0% { transform: rotate(0deg); } 100% { transform: rotate(360deg); } } .error-message { color: red; font-weight: bold; margin-top: 10px; } .success-message { color: green; font-weight: bold; margin-top: 10px; } /* Responsive Design */ @media (max-width: 600px) { #crawler-container { width: 95%; height: 90%; } .control-buttons { flex-direction: column; } } /* Accessibility Enhancements */ #crawler-container:focus { outline: 2px solid #4caf50; } .crawler-tab:focus, .control-buttons button:focus { outline: 2px solid #4caf50; } `); const container = $(` <div id="crawler-container" tabindex="0"> <div id="crawler-header"> <div id="crawler-tabs"> <div class="crawler-tab" data-tab="logs" title="View Logs" tabindex="0">📜 Logs</div> <div class="crawler-tab" data-tab="data" title="View Collected Data" tabindex="0">📊 Data</div> <div class="crawler-tab" data-tab="settings" title="Change Settings" tabindex="0">⚙️ Settings</div> <div class="crawler-tab" data-tab="previous-crawls" title="Previous Crawling Records" tabindex="0">📁 Records</div> </div> <button id="crawler-minimize" title="Minimize Window" aria-label="Minimize Window">─</button> </div> <div id="crawler-content"> <div id="tab-logs" style="display: none;"> <div class="progress-bar" aria-label="Crawling Progress"> <div class="progress-bar-fill" style="width: 0%;" role="progressbar" aria-valuemin="0" aria-valuemax="100" aria-valuenow="0"></div> </div> <div class="status-text">Progress: 0% (0/0 pages)</div> <div class="control-buttons"> <button id="start-crawling" title="Start Crawling">▶ Start</button> <button id="stop-crawling" title="Stop Crawling" disabled>■ Stop</button> ${Config.settings.exportFormats.csv ? `<button id="download-csv" title="Download CSV">📥 CSV</button>` : ''} ${Config.settings.exportFormats.json ? `<button id="download-json" title="Download JSON">📥 JSON</button>` : ''} ${Config.settings.exportFormats.xml ? `<button id="download-xml" title="Download XML">📥 XML</button>` : ''} ${Config.settings.exportFormats.xls ? `<button id="download-xls" title="Download XLS">📥 XLS</button>` : ''} <button id="reset-data" title="Reset All Crawling Data">🗑️ Reset</button> </div> <div id="log-messages" style="height: 200px; overflow-y: auto; background: #f9f9f9; padding: 10px; border: 1px solid #ccc; border-radius: 4px;" aria-live="polite"> Logs will appear here. </div> <div id="error-message" class="error-message" style="display: none;" role="alert"></div> <div id="success-message" class="success-message" style="display: none;" role="alert"></div> </div> <div id="tab-data" style="display: none;"> <input type="text" id="search-filter" placeholder="Search Filter..." aria-label="Search Filter"> <table> <thead> <tr> ${Config.settings.dataFields.name ? '<th data-sort="name">Site Name</th>' : ''} ${Config.settings.dataFields.url ? '<th data-sort="url">URL</th>' : ''} ${Config.settings.dataFields.description ? '<th data-sort="description">Description</th>' : ''} ${Config.settings.dataFields.html ? '<th>📄 HTML Content</th>' : ''} ${Config.settings.dataFields.category ? '<th data-sort="category">Category</th>' : ''} ${Config.settings.dataFields.quality ? '<th data-sort="quality">Quality Score</th>' : ''} ${Config.settings.dataFields.language ? '<th data-sort="language">Language</th>' : ''} ${Config.settings.dataFields.wordCount ? '<th data-sort="wordCount">Word Count</th>' : ''} ${Config.settings.dataFields.metaTags ? '<th>Meta Tags</th>' : ''} ${Config.settings.dataFields.headings ? '<th>Headings</th>' : ''} </tr> </thead> <tbody id="data-table-body"></tbody> </table> ${Config.settings.dataVisualization ? ` <div id="data-visualization"> <h3>Data Visualization</h3> <canvas id="data-chart" aria-label="Data Chart"></canvas> </div> ` : ''} </div> <div id="tab-settings" style="display: none;"> <h3>Settings</h3> <form id="settings-form"> <div class="settings-section"> <label for="delay-input">Crawling Delay (ms): <span class="tooltip">ℹ️ <span class="tooltiptext">Sets the delay between requests by combining the base delay and random delay.</span> </span> </label> <input type="number" id="delay-input" value="${Config.settings.delay}" min="500" step="500" required> </div> <div class="settings-section"> <label for="delay-random-input">Random Delay Range (ms): <span class="tooltip">ℹ️ <span class="tooltiptext">Adds a random element to the delay for more natural requests.</span> </span> </label> <input type="number" id="delay-random-input" value="${Config.settings.delayRandom}" min="0" step="500" required> </div> <div class="settings-section"> <label for="max-pages-input">Maximum Crawling Pages: <span class="tooltip">ℹ️ <span class="tooltiptext">Sets the maximum number of pages to crawl.</span> </span> </label> <input type="number" id="max-pages-input" value="${Config.settings.maxPages}" min="1" required> </div> <div class="settings-section"> <label for="user-agent-select">User-Agent: <span class="tooltip">ℹ️ <span class="tooltiptext">Select or add a User-Agent for crawling requests.</span> </span> </label> <select id="user-agent-select" multiple size="5" aria-label="User-Agent Select"> ${Config.settings.userAgents.map((ua, idx) => `<option value="${idx}" ${idx === Config.settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`).join('')} </select> <div style="margin-top:5px;"> <button type="button" id="add-user-agent" style="padding: 5px 10px; background-color: #2196F3; color: white; border: none; border-radius: 4px; font-size: 12px;">Add</button> <button type="button" id="remove-user-agent" style="padding: 5px 10px; background-color: #f44336; color: white; border: none; border-radius: 4px; font-size: 12px;">Remove</button> </div> </div> <div class="settings-section"> <label>Content Types:</label> <div> <strong>Include:</strong> <textarea id="content-include-input" rows="2" aria-label="Include Content Types">${Config.settings.contentTypes.include.join(', ')}</textarea> </div> <div> <strong>Exclude:</strong> <textarea id="content-exclude-input" rows="2" aria-label="Exclude Content Types">${Config.settings.contentTypes.exclude.join(', ')}</textarea> </div> </div> <div class="settings-section"> <label for="timeout-input">Crawling Timeout (ms): <span class="tooltip">ℹ️ <span class="tooltiptext">Sets the maximum time to wait for a request to complete.</span> </span> </label> <input type="number" id="timeout-input" value="${Config.settings.timeout}" min="1000" step="1000" required> </div> <div class="settings-section"> <label for="include-keywords-input">Include Keywords: <span class="tooltip">ℹ️ <span class="tooltiptext">Keywords that must be present in the crawled pages, separated by newlines.</span> </span> </label> <textarea id="include-keywords-input" rows="3" aria-label="Include Keywords">${Config.settings.includeKeywords.join('\n')}</textarea> </div> <div class="settings-section"> <label for="exclude-keywords-input">Exclude Keywords: <span class="tooltip">ℹ️ <span class="tooltiptext">Keywords that must not be present in the crawled pages, separated by newlines.</span> </span> </label> <textarea id="exclude-keywords-input" rows="3" aria-label="Exclude Keywords">${Config.settings.excludeKeywords.join('\n')}</textarea> </div> <div class="settings-section"> <label for="regex-include-input">Include Keywords (Regex): <span class="tooltip">ℹ️ <span class="tooltiptext">Regular expressions for keywords to include, separated by newlines.</span> </span> </label> <textarea id="regex-include-input" rows="3" aria-label="Include Keywords Regex">${Config.settings.regexInclude.join('\n')}</textarea> </div> <div class="settings-section"> <label for="regex-exclude-input">Exclude Keywords (Regex): <span class="tooltip">ℹ️ <span class="tooltiptext">Regular expressions for keywords to exclude, separated by newlines.</span> </span> </label> <textarea id="regex-exclude-input" rows="3" aria-label="Exclude Keywords Regex">${Config.settings.regexExclude.join('\n')}</textarea> </div> <div class="settings-section"> <label for="language-select">Language Filtering:</label> <select id="language-select" multiple size="10" aria-label="Language Filtering"> ${[ { value: 'en', text: 'English (en)' }, { value: 'ko', text: 'Korean (ko)' }, { value: 'zh', text: 'Chinese (zh)' }, { value: 'ja', text: 'Japanese (ja)' }, { value: 'ru', text: 'Russian (ru)' }, { value: 'es', text: 'Spanish (es)' }, { value: 'fr', text: 'French (fr)' }, { value: 'de', text: 'German (de)' }, { value: 'it', text: 'Italian (it)' }, { value: 'pt', text: 'Portuguese (pt)' }, { value: 'ar', text: 'Arabic (ar)' }, { value: 'hi', text: 'Hindi (hi)' }, { value: 'bn', text: 'Bengali (bn)' }, { value: 'pa', text: 'Punjabi (pa)' }, { value: 'nl', text: 'Dutch (nl)' }, { value: 'sv', text: 'Swedish (sv)' }, { value: 'tr', text: 'Turkish (tr)' }, { value: 'vi', text: 'Vietnamese (vi)' }, { value: 'pl', text: 'Polish (pl)' }, { value: 'th', text: 'Thai (th)' } ].map(lang => `<option value="${lang.value}" ${Config.settings.language.includes(lang.value) ? 'selected' : ''}>${lang.text}</option>`).join('')} </select> </div> <div class="settings-section"> <label for="min-word-count-input">Minimum Word Count: <span class="tooltip">ℹ️ <span class="tooltiptext">Sets the minimum number of words on the crawled pages.</span> </span> </label> <input type="number" id="min-word-count-input" value="${Config.settings.minWordCount}" min="10" step="10" required> <label for="max-word-count-input">Maximum Word Count: <span class="tooltip">ℹ️ <span class="tooltiptext">Sets the maximum number of words on the crawled pages.</span> </span> </label> <input type="number" id="max-word-count-input" value="${Config.settings.maxWordCount}" min="100" step="100" required> </div> <div class="settings-section"> <label for="max-depth-input">Maximum Crawling Depth: <span class="tooltip">ℹ️ <span class="tooltiptext">Sets the maximum depth for following links.</span> </span> </label> <input type="number" id="max-depth-input" value="${Config.settings.maxDepth}" min="1" step="1" required> </div> <div class="settings-section"> <label for="proxy-input">Proxy Server: <span class="tooltip">ℹ️ <span class="tooltiptext">Enter the proxy server address. Example: http://proxyserver:port</span> </span> </label> <input type="text" id="proxy-input" value="${Config.settings.proxy}" placeholder="http://proxyserver:port" aria-label="Proxy Server"> </div> <div class="settings-section"> <label>Data Fields to Collect:</label> <div class="checkbox-group"> ${Object.keys(Config.settings.dataFields).map(field => `<label><input type="checkbox" id="field-${field}" ${Config.settings.dataFields[field] ? 'checked' : ''}> ${Utils.capitalize(getFieldName(field))}</label>`).join('')} </div> </div> <div class="settings-section"> <label for="retry-attempts-input">Retry Attempts: <span class="tooltip">ℹ️ <span class="tooltiptext">Sets the number of retry attempts for failed requests.</span> </span> </label> <input type="number" id="retry-attempts-input" value="${Config.settings.retryAttempts}" min="0" step="1" required> <label for="retry-delay-input">Retry Delay (ms): <span class="tooltip">ℹ️ <span class="tooltiptext">Sets the delay between retry attempts in milliseconds.</span> </span> </label> <input type="number" id="retry-delay-input" value="${Config.settings.retryDelay}" min="1000" step="500" required> </div> <div class="settings-section"> <label>Crawling Scheduling:</label> <div class="checkbox-group"> <label><input type="checkbox" id="schedule-enabled" ${Config.settings.schedule.enabled ? 'checked' : ''}> Enable Scheduling</label> </div> <div id="schedule-options" style="display: ${Config.settings.schedule.enabled ? 'block' : 'none'}; margin-top: 10px;"> <label for="start-time-input">Start Time (HH:MM): <span class="tooltip">ℹ️ <span class="tooltiptext">Sets the time to start crawling.</span> </span> </label> <input type="time" id="start-time-input" value="${Config.settings.schedule.startTime}" aria-label="Start Time"> <label for="interval-input">Crawling Interval (minutes): <span class="tooltip">ℹ️ <span class="tooltiptext">Sets the interval between crawling sessions in minutes.</span> </span> </label> <input type="number" id="interval-input" value="${Config.settings.schedule.interval / 60000}" min="1" step="1" aria-label="Crawling Interval"> </div> </div> <div class="settings-section"> <label>Crawling Search Engines:</label> <div class="checkbox-group"> ${Object.keys(Config.settings.searchEngines).map(engine => `<label><input type="checkbox" id="engine-${engine}" ${Config.settings.searchEngines[engine] ? 'checked' : ''}> ${Utils.capitalize(engine)}</label>`).join('')} </div> </div> <div class="settings-section"> <label for="blacklist-input">Blacklist Domains: <span class="tooltip">ℹ️ <span class="tooltiptext">Domains to exclude from crawling, separated by newlines.</span> </span> </label> <textarea id="blacklist-input" rows="5" aria-label="Blacklist Domains">${Config.settings.blacklist.join('\n')}</textarea> </div> <div class="settings-section"> <label for="whitelist-input">Whitelist Domains: <span class="tooltip">ℹ️ <span class="tooltiptext">Domains to allow crawling, separated by newlines.</span> </span> </label> <textarea id="whitelist-input" rows="5" aria-label="Whitelist Domains">${Config.settings.whitelist.join('\n')}</textarea> </div> <div class="settings-section"> <label for="respect-robots-checkbox">Respect robots.txt: <span class="tooltip">ℹ️ <span class="tooltiptext">Crawling adheres to the site's robots.txt rules.</span> </span> </label> <input type="checkbox" id="respect-robots-checkbox" ${Config.settings.respectRobots ? 'checked' : ''} aria-label="Respect robots.txt"> </div> <div class="settings-section"> <label for="max-concurrent-requests-input">Max Concurrent Requests: <span class="tooltip">ℹ️ <span class="tooltiptext">Sets the maximum number of concurrent requests.</span> </span> </label> <input type="number" id="max-concurrent-requests-input" value="${Config.settings.maxConcurrentRequests}" min="1" step="1" required> </div> <div class="settings-section"> <label for="data-visualization-checkbox">Enable Data Visualization: <span class="tooltip">ℹ️ <span class="tooltiptext">Visualizes collected data in charts.</span> </span> </label> <input type="checkbox" id="data-visualization-checkbox" ${Config.settings.dataVisualization ? 'checked' : ''} aria-label="Enable Data Visualization"> </div> <div class="settings-section"> <label for="enable-captchas-checkbox">Enable CAPTCHA Handling: <span class="tooltip">ℹ️ <span class="tooltiptext">Automatically handles CAPTCHA challenges when detected. (Limited)</span> </span> </label> <input type="checkbox" id="enable-captchas-checkbox" ${Config.settings.enableCaptchas ? 'checked' : ''} aria-label="Enable CAPTCHA Handling"> </div> <div class="settings-section"> <label>Data Export Formats:</label> <div class="checkbox-group"> ${Object.keys(Config.settings.exportFormats).map(format => `<label><input type="checkbox" id="export-${format}" ${Config.settings.exportFormats[format] ? 'checked' : ''}> ${format.toUpperCase()}</label>`).join('')} </div> </div> <div class="settings-section"> <label for="advanced-logging-checkbox">Enable Advanced Logging: <span class="tooltip">ℹ️ <span class="tooltiptext">Enables detailed logging for debugging and monitoring.</span> </span> </label> <input type="checkbox" id="advanced-logging-checkbox" ${Config.settings.advancedLogging ? 'checked' : ''} aria-label="Enable Advanced Logging"> </div> <div class="settings-section"> <label for="custom-keywords-input">Custom Keyword Classification: <span class="tooltip">ℹ️ <span class="tooltiptext">Add custom keywords for enhanced data classification. Format: Category:Keyword1,Keyword2,...</span> </span> </label> <textarea id="custom-keywords-input" rows="5" aria-label="Custom Keyword Classification">${Utils.formatCustomKeywords(Config.settings.customKeywords)}</textarea> </div> <button type="button" id="save-settings" style="padding: 10px 20px; background-color: #4caf50; color: white; border: none; border-radius: 4px; font-size: 14px;">Save</button> <button type="button" id="reset-settings" style="padding: 10px 20px; background-color: #f44336; color: white; border: none; border-radius: 4px; font-size: 14px;">Reset</button> </form> </div> <div id="tab-previous-crawls" style="display: none;"> <h3>Previous Crawling Records</h3> <button id="clear-previous-crawls" style="padding: 8px 12px; background-color: #f44336; color: white; border: none; border-radius: 4px; font-size: 14px; margin-bottom: 10px;">Clear Records</button> <table> <thead> <tr> <th>Date</th> <th>URLs Crawled</th> <th>Status</th> </tr> </thead> <tbody id="previous-crawls-body"></tbody> </table> </div> </div> <div id="html-modal" class="modal" aria-hidden="true"> <div class="modal-content"> <span class="close" role="button" aria-label="Close">×</span> <pre id="modal-html-content"></pre> </div> </div> </div> `); $('body').append(container); $('#crawler-minimize').on('click', toggleMinimize); $('.crawler-tab').on('click keypress', function(e) { if (e.type === 'click' || e.key === 'Enter') switchTab.call(this); }); $('#save-settings').on('click', saveSettingsHandler); $('#reset-settings').on('click', resetSettingsHandler); $('#add-user-agent').on('click', addUserAgent); $('#remove-user-agent').on('click', removeUserAgent); $('#download-csv').on('click', Download.downloadCSV); $('#download-json').on('click', Download.downloadJSON); $('#download-xml').on('click', Download.downloadXML); $('#download-xls').on('click', Download.downloadXLS); $('#search-filter').on('input', filterData); $('#schedule-enabled').on('change', toggleScheduleOptions); $('#data-visualization-checkbox').on('change', toggleDataVisualization); $('#enable-captchas-checkbox').on('change', toggleCaptchaHandling); $('.close').on('click', () => $('#html-modal').hide()); $(window).on('click', function(e) { if (e.target.id === 'html-modal') $('#html-modal').hide(); }); $('#start-crawling').on('click', Crawling.startCrawling); $('#stop-crawling').on('click', Crawling.stopCrawling); $('#clear-previous-crawls').on('click', clearPreviousCrawls); $('#reset-data').on('click', resetData); $(document).on('keydown', handleShortcuts); updateUI(); makeDraggable(); } function toggleMinimize() { $('#crawler-content').toggle(); const isHidden = $('#crawler-content').is(':hidden'); $('#crawler-minimize').text(isHidden ? '☐' : '─'); Logger.log(`Crawler window has been ${isHidden ? 'minimized' : 'restored'}.`, 'info'); } function switchTab() { const tab = $(this).data('tab'); $('.crawler-tab').removeClass('active'); $(this).addClass('active'); $('#crawler-content > div').hide(); $(`#tab-${tab}`).show(); Config.saveSettings({ currentTab: tab }); } function updateUI() { (async () => { const currentTab = Config.settings.currentTab || 'logs'; $(`.crawler-tab[data-tab="${currentTab}"]`).addClass('active'); $('#crawler-content > div').hide(); $(`#tab-${currentTab}`).show(); await UI.updateLogs(); await UI.updateDataTable(); await UI.updatePreviousCrawlsUI(); if (Config.settings.dataVisualization) DataVisualization.renderChart(); })(); } function updateLogs() { return new Promise(async (resolve) => { const logs = await Logger.getLogs(); $('#log-messages').html(logs.map(l => `<div>[${l.timestamp}] [${l.type.toUpperCase()}] ${Utils.escape(l.message)}</div>`).join('')); $('.progress-bar-fill').css('width', `${Config.settings.progress.toFixed(2)}%`); $('.progress-bar-fill').attr('aria-valuenow', Config.settings.progress.toFixed(2)); $('.status-text').text(`Progress: ${Config.settings.progress.toFixed(2)}% (${Config.settings.currentPage}/${Config.settings.maxPages} pages)`); const latestLog = logs[logs.length -1]; if (latestLog) { if (latestLog.type === 'error') { $('#error-message').text(latestLog.message).show(); setTimeout(() => { $('#error-message').fadeOut(); }, 5000); } if (latestLog.type === 'success') { $('#success-message').text(latestLog.message).show(); setTimeout(() => { $('#success-message').fadeOut(); }, 5000); } } resolve(); }); } async function updateDataTable() { const data = await DB.getAllData(); const filter = $('#search-filter').val().toLowerCase(); const filtered = filter ? data.filter(item => Object.keys(Config.settings.dataFields).some(field => Config.settings.dataFields[field] && String(item[field]).toLowerCase().includes(filter))) : data; const rows = filtered.map(item => ` <tr> ${Config.settings.dataFields.name ? `<td>${Utils.escape(item.name)}</td>` : ''} ${Config.settings.dataFields.url ? `<td><a href="${Utils.escape(item.url)}" target="_blank" rel="noopener noreferrer">${Utils.escape(item.url)}</a></td>` : ''} ${Config.settings.dataFields.description ? `<td>${Utils.escape(item.description)}</td>` : ''} ${Config.settings.dataFields.html ? `<td><button class="view-html" data-html="${encodeURIComponent(item.html)}" aria-label="View HTML Content">📄 View</button></td>` : ''} ${Config.settings.dataFields.category ? `<td>${Utils.escape(item.category)}</td>` : ''} ${Config.settings.dataFields.quality ? `<td>${Utils.escape(item.quality.toString())}</td>` : ''} ${Config.settings.dataFields.language ? `<td>${Utils.escape(item.language)}</td>` : ''} ${Config.settings.dataFields.wordCount ? `<td>${Utils.escape(item.wordCount.toString())}</td>` : ''} ${Config.settings.dataFields.metaTags ? `<td>${Utils.escape(JSON.stringify(item.metaTags))}</td>` : ''} ${Config.settings.dataFields.headings ? `<td>${Utils.escape(item.headings.join(', '))}</td>` : ''} </tr> `).join(''); $('#data-table-body').html(rows); $('.view-html').off('click').on('click', function() { $('#modal-html-content').text(decodeURIComponent($(this).data('html'))); $('#html-modal').show(); $('#html-modal').attr('aria-hidden', 'false'); }); if (Config.settings.dataVisualization) DataVisualization.renderChart(); } async function updatePreviousCrawlsUI() { const previousCrawls = await DB.getPreviousCrawls(); $('#previous-crawls-body').html(previousCrawls.map(crawl => ` <tr> <td>${Utils.escape(crawl.date)}</td> <td>${Utils.escape(crawl.count.toString())}</td> <td>${Utils.escape(crawl.status)}</td> </tr> `).join('')); } function addUserAgent() { const ua = prompt("Enter a new User-Agent:"); if (ua) { Config.settings.userAgents.push(ua); Config.saveSettings({ userAgents: Config.settings.userAgents }); updateUserAgentSelect(); Logger.log('A new User-Agent has been added.', 'success'); } } function removeUserAgent() { const selected = $('#user-agent-select').val(); if (selected && selected.length) { selected.sort((a, b) => b - a).forEach(idx => { Config.settings.userAgents.splice(idx, 1); }); Config.settings.userAgentIndex = 0; Config.saveSettings({ userAgents: Config.settings.userAgents, userAgentIndex: Config.settings.userAgentIndex }); updateUserAgentSelect(); Logger.log('Selected User-Agent(s) have been removed.', 'success'); } else { alert('Please select User-Agent(s) to remove.'); } } function updateUserAgentSelect() { $('#user-agent-select').empty(); Config.settings.userAgents.forEach((ua, idx) => { $('#user-agent-select').append(`<option value="${idx}" ${idx === Config.settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`); }); } async function saveSettingsHandler() { const form = $('#settings-form')[0]; if (!form.checkValidity()) { $('#error-message').text('Please fill out all required fields with valid values.').show(); setTimeout(() => { $('#error-message').fadeOut(); }, 5000); return; } const delay = parseInt($('#delay-input').val(), 10); const delayRandom = parseInt($('#delay-random-input').val(), 10); const maxPages = parseInt($('#max-pages-input').val(), 10); const timeout = parseInt($('#timeout-input').val(), 10); const minWordCount = parseInt($('#min-word-count-input').val(), 10); const maxWordCount = parseInt($('#max-word-count-input').val(), 10); const maxDepth = parseInt($('#max-depth-input').val(), 10); const retryAttempts = parseInt($('#retry-attempts-input').val(), 10); const retryDelay = parseInt($('#retry-delay-input').val(), 10); const interval = parseInt($('#interval-input').val(), 10) * 60000; if (!isValidSettings(delay, delayRandom, maxPages, timeout, minWordCount, maxWordCount, maxDepth, retryAttempts, retryDelay, interval)) { $('#error-message').text('Invalid settings values. Please check your inputs.').show(); setTimeout(() => { $('#error-message').fadeOut(); }, 5000); return; } Config.settings.delay = delay; Config.settings.delayRandom = delayRandom; Config.settings.maxPages = maxPages; const selectedUA = parseInt($('#user-agent-select').val(), 10); Config.settings.userAgentIndex = isNaN(selectedUA) ? 0 : selectedUA; Config.settings.contentTypes.include = $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(ct => ct); Config.settings.contentTypes.exclude = $('#content-exclude-input').val().split(',').map(ct => ct.trim()).filter(ct => ct); Config.settings.timeout = timeout; Config.settings.includeKeywords = $('#include-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw); Config.settings.excludeKeywords = $('#exclude-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw); Config.settings.regexInclude = $('#regex-include-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx); Config.settings.regexExclude = $('#regex-exclude-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx); Config.settings.language = $('#language-select').val() || []; Config.settings.minWordCount = minWordCount; Config.settings.maxWordCount = maxWordCount; Config.settings.maxDepth = maxDepth; Config.settings.proxy = $('#proxy-input').val().trim() || Config.DEFAULT_SETTINGS.proxy; ['name','url','description','html','category','quality','language','wordCount','metaTags','headings'].forEach(field => { Config.settings.dataFields[field] = $(`#field-${field}`).is(':checked'); }); Config.settings.retryAttempts = retryAttempts; Config.settings.retryDelay = retryDelay; Config.settings.schedule.enabled = $('#schedule-enabled').is(':checked'); Config.settings.schedule.startTime = $('#start-time-input').val() || Config.DEFAULT_SETTINGS.schedule.startTime; Config.settings.schedule.interval = Config.settings.schedule.enabled ? (isNaN(interval) ? Config.DEFAULT_SETTINGS.schedule.interval : interval) : Config.DEFAULT_SETTINGS.schedule.interval; Object.keys(Config.settings.searchEngines).forEach(engine => { Config.settings.searchEngines[engine] = $(`#engine-${engine}`).is(':checked'); }); Config.settings.blacklist = $('#blacklist-input').val().split('\n').map(d => d.trim()).filter(d => d); Config.settings.whitelist = $('#whitelist-input').val().split('\n').map(d => d.trim()).filter(d => d); Config.settings.respectRobots = $('#respect-robots-checkbox').is(':checked'); Config.settings.maxConcurrentRequests = parseInt($('#max-concurrent-requests-input').val(), 10) || Config.DEFAULT_SETTINGS.maxConcurrentRequests; Config.settings.dataVisualization = $('#data-visualization-checkbox').is(':checked'); Config.settings.exportFormats.csv = $('#export-csv').is(':checked'); Config.settings.exportFormats.json = $('#export-json').is(':checked'); Config.settings.exportFormats.xml = $('#export-xml').is(':checked'); Config.settings.exportFormats.xls = $('#export-xls').is(':checked'); Config.settings.advancedLogging = $('#advanced-logging-checkbox').is(':checked'); Config.settings.enableCaptchas = $('#enable-captchas-checkbox').is(':checked'); Config.settings.customKeywords = Utils.parseCustomKeywords($('#custom-keywords-input').val()); Config.saveSettings(Config.settings); Crawling.updateCrawlSettings(); Logger.log('Settings have been saved.', 'success'); if (Config.settings.schedule.enabled) { Scheduler.scheduleCrawling(); } else { Scheduler.clearSchedule(); } await updateDataTable(); if (Config.settings.dataVisualization) DataVisualization.renderChart(); } function isValidSettings(delay, delayRandom, maxPages, timeout, minWordCount, maxWordCount, maxDepth, retryAttempts, retryDelay, interval) { return !isNaN(delay) && delay >= 500 && !isNaN(delayRandom) && delayRandom >= 0 && !isNaN(maxPages) && maxPages >= 1 && !isNaN(timeout) && timeout >= 1000 && !isNaN(minWordCount) && minWordCount >= 10 && !isNaN(maxWordCount) && maxWordCount >= 100 && !isNaN(maxDepth) && maxDepth >= 1 && !isNaN(retryAttempts) && retryAttempts >= 0 && !isNaN(retryDelay) && retryDelay >= 1000 && (!Config.settings.schedule.enabled || (!isNaN(interval) && interval >= 60000)); } function resetSettingsHandler() { if (confirm("Are you sure you want to reset to default settings? This will also clear all crawling data.")) { Config.resetSettings(); DB.clearData(); DB.clearLogs(); DB.clearVisited(); DB.clearPreviousCrawls(); createUI(); Logger.log('Settings have been reset to default.', 'success'); } } function toggleScheduleOptions() { $('#schedule-options').toggle($('#schedule-enabled').is(':checked')); } function toggleDataVisualization() { Config.settings.dataVisualization = $('#data-visualization-checkbox').is(':checked'); Config.saveSettings({ dataVisualization: Config.settings.dataVisualization }); $('#data-visualization').toggle(Config.settings.dataVisualization); Logger.log('Data visualization settings have been updated.', 'success'); if (Config.settings.dataVisualization) DataVisualization.renderChart(); } function toggleCaptchaHandling() { Config.settings.enableCaptchas = $('#enable-captchas-checkbox').is(':checked'); Config.saveSettings({ enableCaptchas: Config.settings.enableCaptchas }); Logger.log('CAPTCHA handling settings have been updated.', 'success'); } function filterData() { updateDataTable(); } async function clearPreviousCrawls() { if (confirm("Are you sure you want to clear all previous crawling records?")) { await DB.clearPreviousCrawls(); updatePreviousCrawlsUI(); Logger.log('Previous crawling records have been cleared.', 'success'); } } async function resetData() { if (confirm("Are you sure you want to reset all crawling data?")) { await DB.clearData(); await DB.clearVisited(); updateDataTable(); Logger.log('Crawling data has been reset.', 'success'); } } function handleShortcuts(e) { if (e.ctrlKey) { const actions = { '1': 'logs', '2': 'data', '3': 'settings', '4': 'previous-crawls', 's': 'save', 'r': 'reset', 'p': 'start', 'q': 'stop', 'c': 'csv', 'j': 'json', 'x': 'xml', 'l': 'xls' }; const action = actions[e.key]; if (action) { e.preventDefault(); switch(action) { case 'logs': $('.crawler-tab[data-tab="logs"]').click(); break; case 'data': $('.crawler-tab[data-tab="data"]').click(); break; case 'settings': $('.crawler-tab[data-tab="settings"]').click(); break; case 'previous-crawls': $('.crawler-tab[data-tab="previous-crawls"]').click(); break; case 'save': saveSettingsHandler(); break; case 'reset': resetSettingsHandler(); break; case 'start': if (!Crawling.isCrawling) Crawling.startCrawling(); break; case 'stop': if (Crawling.isCrawling) Crawling.stopCrawling(); break; case 'csv': Config.settings.exportFormats.csv && Download.downloadCSV(); break; case 'json': Config.settings.exportFormats.json && Download.downloadJSON(); break; case 'xml': Config.settings.exportFormats.xml && Download.downloadXML(); break; case 'xls': Config.settings.exportFormats.xls && Download.downloadXLS(); break; } } } } function makeDraggable() { const container = document.getElementById('crawler-container'); const header = document.getElementById('crawler-header'); let isDragging = false; let startX, startY, initialX, initialY; header.onmousedown = function(e) { isDragging = true; startX = e.clientX; startY = e.clientY; initialX = container.offsetLeft; initialY = container.offsetTop; document.onmousemove = drag; document.onmouseup = stopDrag; return false; }; function drag(e) { if (isDragging) { let dx = e.clientX - startX; let dy = e.clientY - startY; container.style.left = (initialX + dx) + 'px'; container.style.top = (initialY + dy) + 'px'; } } function stopDrag() { isDragging = false; document.onmousemove = null; document.onmouseup = null; } } function getFieldName(field) { const names = { name: 'Name', url: 'URL', description: 'Description', html: 'HTML Content', category: 'Category', quality: 'Quality Score', language: 'Language', wordCount: 'Word Count', metaTags: 'Meta Tags', headings: 'Headings' }; return names[field] || field; } return { createUI, updateLogs, updateDataTable, updatePreviousCrawlsUI }; })(); // Download Module const Download = (() => { async function downloadCSV() { const data = await DB.getAllData(); if (!data.length) { Logger.log('No data available for download.', 'warning'); return; } const headers = Object.keys(Config.settings.dataFields).filter(f => Config.settings.dataFields[f]).map(f => Utils.capitalize(getFieldName(f))); const csvRows = [headers.join(",")]; data.forEach(item => { let row = []; Object.keys(Config.settings.dataFields).forEach(field => { if (Config.settings.dataFields[field]) row.push(`"${Utils.escape(item[field] || '', 'csv')}"`); }); csvRows.push(row.join(",")); }); const csvContent = csvRows.join("\n"); const blob = new Blob([csvContent], { type: "text/csv;charset=utf-8;" }); const link = document.createElement("a"); const url = URL.createObjectURL(blob); link.href = url; link.download = "data.csv"; document.body.appendChild(link); link.click(); document.body.removeChild(link); URL.revokeObjectURL(url); Logger.log('CSV file has been downloaded.', 'success'); } async function downloadJSON() { const data = await DB.getAllData(); if (!data.length) { Logger.log('No data available for download.', 'warning'); return; } const json = JSON.stringify(data, null, 2); const blob = new Blob([json], { type: "application/json" }); const link = document.createElement("a"); const url = URL.createObjectURL(blob); link.href = url; link.download = "data.json"; document.body.appendChild(link); link.click(); document.body.removeChild(link); URL.revokeObjectURL(url); Logger.log('JSON file has been downloaded.', 'success'); } async function downloadXML() { const data = await DB.getAllData(); if (!data.length) { Logger.log('No data available for download.', 'warning'); return; } let xml = '<?xml version="1.0" encoding="UTF-8"?><data>'; data.forEach(item => { xml += '<entry>'; if (Config.settings.dataFields.name) xml += `<name>${Utils.escape(item.name || 'No Title', 'xml')}</name>`; if (Config.settings.dataFields.url) xml += `<url>${Utils.escape(item.url || '', 'xml')}</url>`; if (Config.settings.dataFields.description) xml += `<description>${Utils.escape(item.description || 'No Description', 'xml')}</description>`; if (Config.settings.dataFields.html) xml += `<html><![CDATA[${item.html || ''}]]></html>`; if (Config.settings.dataFields.category) xml += `<category>${Utils.escape(item.category || '', 'xml')}</category>`; if (Config.settings.dataFields.quality) xml += `<quality>${Utils.escape(item.quality)}</quality>`; if (Config.settings.dataFields.language) xml += `<language>${Utils.escape(item.language || 'unknown', 'xml')}</language>`; if (Config.settings.dataFields.wordCount) xml += `<wordCount>${Utils.escape(item.wordCount)}</wordCount>`; if (Config.settings.dataFields.metaTags) xml += `<metaTags>${Utils.escape(JSON.stringify(item.metaTags), 'xml')}</metaTags>`; if (Config.settings.dataFields.headings) xml += `<headings>${Utils.escape(item.headings.join('; '), 'xml')}</headings>`; xml += '</entry>'; }); xml += '</data>'; const blob = new Blob([xml], { type: "application/xml" }); const link = document.createElement("a"); const url = URL.createObjectURL(blob); link.href = url; link.download = "data.xml"; document.body.appendChild(link); link.click(); document.body.removeChild(link); URL.revokeObjectURL(url); Logger.log('XML file has been downloaded.', 'success'); } async function downloadXLS() { const data = await DB.getAllData(); if (!data.length) { Logger.log('No data available for download.', 'warning'); return; } const sheetData = []; const headers = Object.keys(Config.settings.dataFields).filter(f => Config.settings.dataFields[f]).map(f => Utils.capitalize(getFieldName(f))); sheetData.push(headers); data.forEach(item => { let row = []; Object.keys(Config.settings.dataFields).forEach(field => { if (Config.settings.dataFields[field]) { if (field === 'metaTags') row.push(JSON.stringify(item[field])); else if (field === 'headings') row.push(item[field].join('; ')); else row.push(item[field] || ''); } }); sheetData.push(row); }); const sheet = XLSX.utils.aoa_to_sheet(sheetData); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, sheet, "Data"); XLSX.writeFile(wb, "data.xlsx"); Logger.log('XLS file has been downloaded.', 'success'); } function getFieldName(field) { const names = { name: 'Name', url: 'URL', description: 'Description', html: 'HTML Content', category: 'Category', quality: 'Quality Score', language: 'Language', wordCount: 'Word Count', metaTags: 'Meta Tags', headings: 'Headings' }; return names[field] || field; } return { downloadCSV, downloadJSON, downloadXML, downloadXLS }; })(); // Data Visualization Module const DataVisualization = (() => { let chartInstance = null; async function renderChart() { if (!document.getElementById('data-chart').getContext) return; const ctx = document.getElementById('data-chart').getContext('2d'); if (chartInstance) chartInstance.destroy(); const data = await DB.getAllData(); const categories = {}; data.forEach(item => { const cat = item.category || 'Other'; categories[cat] = (categories[cat] || 0) + 1; }); const chartData = { labels: Object.keys(categories), datasets: [{ label: 'Number of Data per Category', data: Object.values(categories), backgroundColor: Utils.generateColors(Object.keys(categories).length) }] }; chartInstance = new Chart(ctx, { type: 'pie', data: chartData, options: { responsive: true } }); } return { renderChart }; })(); // Crawling Module const Crawling = (() => { let isCrawling = Config.settings.isCrawling; let currentPage = Config.settings.currentPage; let totalPages = Config.settings.maxPages; let progress = Config.settings.progress; let currentDepth = Config.settings.currentDepth; let activeRequests = 0; let requestQueue = []; let previousCrawls = await DB.getPreviousCrawls(); const searchEngineHandlers = { google: { extractResults: function(doc) { const results = []; doc.querySelectorAll('div.g, div[data-hveid]').forEach(item => { const name = item.querySelector('h3')?.innerText.trim() || 'No Title'; const url = item.querySelector('a')?.href || ''; const desc = item.querySelector('.aCOpRe, .VwiC3b')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: function(doc) { return doc.querySelector('a#pnnext')?.href || null; } }, bing: { extractResults: function(doc) { const results = []; doc.querySelectorAll('.b_algo').forEach(item => { const name = item.querySelector('h2')?.innerText.trim() || 'No Title'; const url = item.querySelector('h2 a')?.href || ''; const desc = item.querySelector('.b_caption p')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: function(doc) { return doc.querySelector('a.sb_pagN')?.href || null; } }, duckduckgo: { extractResults: function(doc) { const results = []; doc.querySelectorAll('.result').forEach(item => { const name = item.querySelector('.result__title')?.innerText.trim() || 'No Title'; const url = item.querySelector('.result__url')?.href || ''; const desc = item.querySelector('.result__snippet')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: function(doc) { return doc.querySelector('a.result--more__btn')?.href || null; } }, yahoo: { extractResults: function(doc) { const results = []; doc.querySelectorAll('.dd algo-sr').forEach(item => { const name = item.querySelector('h3')?.innerText.trim() || 'No Title'; const url = item.querySelector('a')?.href || ''; const desc = item.querySelector('.compText a')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: function(doc) { return doc.querySelector('a.next')?.href || null; } }, // Additional Search Engine Handlers... }; function identifySearchEngine() { const host = window.location.hostname; if (host.includes('google.com')) return 'google'; if (host.includes('bing.com')) return 'bing'; if (host.includes('yahoo.com')) return 'yahoo'; if (host.includes('duckduckgo.com')) return 'duckduckgo'; // Add other search engines... return null; } async function startCrawling() { if (isCrawling) { Logger.log('Crawling is already in progress.', 'warning'); return; } if (!Object.values(Config.settings.searchEngines).some(v => v)) { Logger.log('No search engines selected for crawling.', 'error'); return; } isCrawling = true; Config.saveSettings({ isCrawling }); await DB.addPreviousCrawl({ date: new Date().toLocaleString(), count: 0, status: 'In Progress' }); previousCrawls = await DB.getPreviousCrawls(); Logger.log('Crawling has started.', 'info'); UI.updateLogs(); currentPage = Config.settings.currentPage; totalPages = Config.settings.maxPages; currentDepth = Config.settings.currentDepth; setButtonStates(); await crawlPage(currentDepth); } async function stopCrawling() { if (!isCrawling) { Logger.log('Crawling is not in progress.', 'warning'); return; } isCrawling = false; Config.saveSettings({ isCrawling }); Logger.log('Crawling has been stopped.', 'warning'); await finalizeCrawl('Stopped'); UI.updateLogs(); setButtonStates(); } function setButtonStates() { $('#start-crawling').prop('disabled', isCrawling); $('#stop-crawling').prop('disabled', !isCrawling); } async function crawlPage(depth) { try { while (isCrawling && currentPage <= totalPages && depth <= Config.settings.maxDepth) { Logger.log(`Crawling page ${currentPage} (Depth: ${depth}).`, 'info'); if (Config.settings.respectRobots && !(await Robots.checkRobotsTxt(window.location.href))) { Logger.log(`Crawling is disallowed by robots.txt for: ${window.location.href}`, 'warning'); break; } const searchEngine = identifySearchEngine(); if (!searchEngine || !searchEngineHandlers[searchEngine]) { Logger.log(`Unsupported search engine: ${window.location.hostname}`, 'error'); break; } const handler = searchEngineHandlers[searchEngine]; const results = handler.extractResults(document); if (!results) { Logger.log('No search results found.', 'error'); break; } const validResults = results.filter(result => { const { url, description } = result; if (await DB.isVisited(url) || !isValidURL(url) || (Config.settings.whitelist.length && !Config.settings.whitelist.some(d => url.includes(d))) || Config.settings.blacklist.some(d => url.includes(d)) || (Config.settings.includeKeywords.length && !Config.settings.includeKeywords.some(k => url.includes(k) || description.includes(k))) || (Config.settings.excludeKeywords.some(k => url.includes(k) || description.includes(k))) || (Config.settings.regexInclude.length && !Config.settings.regexInclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description))) || (Config.settings.regexExclude.length && Config.settings.regexExclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description)))) ) { return false; } return true; }); const uniqueResults = [...new Set(validResults.map(r => r.url))].map(url => validResults.find(r => r.url === url)); requestQueue.push(...uniqueResults.map(r => ({ url: r.url, depth }))); for (const result of uniqueResults) { await DB.addVisited(result.url); } if (previousCrawls.length && previousCrawls[previousCrawls.length -1].status === 'In Progress') { previousCrawls[previousCrawls.length -1].count += uniqueResults.length; await DB.addPreviousCrawl(previousCrawls[previousCrawls.length -1]); UI.updatePreviousCrawlsUI(); } progress = (currentPage / totalPages) * 100; Config.saveSettings({ progress }); UI.updateLogs(); await processQueue(); if (currentPage < totalPages && isCrawling) { const nextUrl = handler.getNextPageUrl(document); if (nextUrl) { currentPage++; Config.saveSettings({ currentPage }); depth++; Config.saveSettings({ currentDepth: depth }); Logger.log(`Navigating to next page: ${currentPage}`, 'info'); const adaptiveDelay = calculateAdaptiveDelay(); await Utils.delay(adaptiveDelay); const nextPageContent = await Fetch.fetchPageContent(nextUrl); if (nextPageContent) { updatePageContent(nextPageContent); continue; } else { Logger.log('Failed to retrieve the next page content.', 'error'); break; } } else { Logger.log('Next page link not found.', 'error'); break; } } else { break; } } if (isCrawling && currentPage > totalPages && activeRequests === 0 && requestQueue.length === 0) { Logger.log('Crawling has been completed.', 'success'); await finalizeCrawl('Completed'); isCrawling = false; Config.saveSettings({ isCrawling }); setButtonStates(); Config.saveSettings({ progress: 100 }); UI.updateLogs(); } } catch (error) { Logger.log(`An error occurred during crawling: ${error.message}`, 'error'); await finalizeCrawl('Error Occurred'); } } async function processQueue() { while (requestQueue.length && activeRequests < Config.settings.maxConcurrentRequests && isCrawling) { const { url, depth } = requestQueue.shift(); activeRequests++; handleRequest(url, depth).finally(() => { activeRequests--; processQueue(); }); } } async function handleRequest(url, depth) { if (!isCrawling || await DB.isVisited(url)) return; try { const response = await Fetch.fetchWithRetry(url, Config.settings.retryAttempts, Config.settings.retryDelay); const contentType = Fetch.getContentType(response); if (!Fetch.isContentTypeAllowed(contentType, Config.settings.contentTypes)) { Logger.log(`Disallowed content type (${contentType}) for URL: ${url}`, 'warning'); return; } const parser = new DOMParser(); const doc = parser.parseFromString(response.responseText, 'text/html'); const text = doc.body ? doc.body.innerText : ''; if (Config.settings.enableCaptchas && Robots.detectCaptcha(doc)) { await Robots.handleCaptcha(url); return; } const wc = text.split(/\s+/).length; if (wc < Config.settings.minWordCount || wc > Config.settings.maxWordCount) { Logger.log(`URL does not meet word count requirements: ${url} (${wc} words)`, 'warning'); return; } const quality = Evaluate.evaluateContentQuality(text); const lang = doc.documentElement.lang || 'unknown'; const category = Classifier.classifyKeywords(text); const meta = Extractor.extractMetaTags(doc); const headings = Extractor.extractHeadings(doc); if (category !== 'Other' || quality > 100) { const entry = { url, quality, language: lang, wordCount: wc, category }; if (Config.settings.dataFields.name) entry.name = Extractor.extractName(doc) || 'No Title'; if (Config.settings.dataFields.description) entry.description = Extractor.extractDescription(doc) || 'No Description'; if (Config.settings.dataFields.html) entry.html = response.responseText; if (Config.settings.dataFields.metaTags) entry.metaTags = meta; if (Config.settings.dataFields.headings) entry.headings = headings; await DB.addData(entry); UI.updateDataTable(); Logger.log(`Data collected for URL: ${url}`, 'success'); if (Config.settings.dataVisualization) DataVisualization.renderChart(); } if (depth < Config.settings.maxDepth) { const links = Extractor.extractLinks(doc); requestQueue.push(...links.map(link => ({ url: link, depth: depth + 1 }))); } } catch (error) { Logger.log(`Failed to collect data for URL: ${url} - ${error.message}`, 'error'); } } async function finalizeCrawl(status) { if (previousCrawls.length) { previousCrawls[previousCrawls.length -1].status = status; await DB.addPreviousCrawl(previousCrawls[previousCrawls.length -1]); UI.updatePreviousCrawlsUI(); } } function calculateAdaptiveDelay() { return Config.settings.delay + Math.random() * Config.settings.delayRandom; } async function updateCrawlSettings() { // Placeholder for any dynamic settings updates during crawling } function updatePageContent(content) { const parser = new DOMParser(); const doc = parser.parseFromString(content, 'text/html'); document.documentElement.innerHTML = doc.documentElement.innerHTML; } function isValidURL(url) { try { new URL(url); return true; } catch { return false; } } return { startCrawling, stopCrawling, isCrawling }; })(); // Fetch Module const Fetch = (() => { async function fetchWithRetry(url, attempts, delayMs) { for (let i = 1; i <= attempts; i++) { try { return await GM_xmlhttpRequestPromise(url); } catch (e) { Logger.log(`Request failed (${i}/${attempts}): ${url} - ${e.message}`, 'warning'); if (i < attempts) await Utils.delay(delayMs); } } throw new Error('All retry attempts failed.'); } function GM_xmlhttpRequestPromise(url) { return new Promise((resolve, reject) => { let reqUrl = Config.settings.proxy ? `${Config.settings.proxy}?url=${encodeURIComponent(url)}` : enforceHTTPS(url); GM_xmlhttpRequest({ method: "GET", url: reqUrl, headers: { 'User-Agent': getCurrentUserAgent(), 'Accept': '*/*' }, timeout: Config.settings.timeout, onload: res => (res.status >= 200 && res.status < 300) ? resolve(res) : reject(new Error(`HTTP ${res.status}`)), onerror: () => reject(new Error('Request failed')), ontimeout: () => reject(new Error('Request timed out')) }); }); } function getCurrentUserAgent() { const index = Config.settings.userAgentIndex % Config.settings.userAgents.length; return Config.settings.userAgents[index]; } function getContentType(res) { return res.responseHeaders.split('\n').find(h => h.toLowerCase().startsWith('content-type'))?.split(':')[1].trim().split(';')[0] || ''; } function isContentTypeAllowed(ct, contentTypes) { if (contentTypes.include.length && !contentTypes.include.includes(ct)) return false; if (contentTypes.exclude.includes(ct)) return false; return true; } async function fetchPageContent(url) { try { const response = await fetchWithRetry(url, Config.settings.retryAttempts, Config.settings.retryDelay); return response.responseText; } catch (error) { Logger.log(`Failed to fetch page content: ${url} - ${error.message}`, 'error'); return null; } } function enforceHTTPS(url) { try { const parsed = new URL(url); parsed.protocol = 'https:'; return parsed.toString(); } catch { return url; } } return { fetchWithRetry, getCurrentUserAgent, getContentType, isContentTypeAllowed, fetchPageContent }; })(); // Robots Module const Robots = (() => { async function checkRobotsTxt(url) { try { const parsed = new URL(url); const robots = `${parsed.origin}/robots.txt`; const res = await Fetch.fetchWithRetry(robots, 2, 2000); return new RobotsParser(res.responseText).isAllowed(url, Fetch.getCurrentUserAgent()); } catch { Logger.log('Failed to check robots.txt. Allowing by default.', 'warning'); return true; } } function detectCaptcha(doc) { return doc.querySelector('iframe[src*="captcha"], form[action*="captcha"]') !== null; } async function handleCaptcha(url) { Logger.log(`CAPTCHA detected: ${url}`, 'warning'); alert(`CAPTCHA detected. Please solve it manually: ${url}`); } class RobotsParser { constructor(text) { this.rules = []; this.parse(text); } parse(text) { let agents = []; text.split('\n').forEach(line => { const [key, ...val] = line.split(':').map(s => s.trim()); if (!key) return; if (key.toLowerCase() === 'user-agent') { agents = [val.join(':').toLowerCase()]; } else if (['disallow','allow'].includes(key.toLowerCase())) this.rules.push({ agents, type: key.toLowerCase(), path: val.join(':') }); }); } isAllowed(url, ua) { const path = new URL(url).pathname; let allowed = true; this.rules.forEach(rule => { if (rule.agents.includes('*') || rule.agents.includes(ua.toLowerCase())) { if (rule.type === 'disallow' && path.startsWith(rule.path)) allowed = false; if (rule.type === 'allow' && path.startsWith(rule.path)) allowed = true; } }); return allowed; } } return { checkRobotsTxt, detectCaptcha, handleCaptcha }; })(); // Extractor Module const Extractor = (() => { function extractMetaTags(doc) { const metas = {}; doc.querySelectorAll('meta[name], meta[property]').forEach(meta => { const name = meta.getAttribute('name') || meta.getAttribute('property'); const content = meta.getAttribute('content'); if (name && content) metas[name.toLowerCase()] = content; }); return metas; } function extractHeadings(doc) { const hs = []; for (let i = 1; i <= 6; i++) doc.querySelectorAll(`h${i}`).forEach(h => hs.push(h.innerText.trim())); return hs; } function extractName(doc) { return doc.querySelector('title')?.innerText || 'No Title'; } function extractDescription(doc) { return doc.querySelector('meta[name="description"]')?.getAttribute('content') || 'No Description'; } function extractLinks(doc) { return Array.from(doc.querySelectorAll('a[href]')).map(a => a.href).filter(isValidURL); } return { extractMetaTags, extractHeadings, extractName, extractDescription, extractLinks }; })(); // Classifier Module const Classifier = (() => { function classifyKeywords(text) { const kws = combineKeywords(); for (const [cat, keywords] of Object.entries(kws)) { if (keywords.some(kw => text.includes(kw))) return cat; } return 'Other'; } function combineKeywords() { const standard = { 'Education': ['school', 'education', 'student', 'professor', 'lecture'], 'Technology': ['technology', 'software', 'hardware', 'programming', 'development'], 'News': ['news', 'breaking', 'article', 'report'], 'Entertainment': ['movie', 'music', 'entertainment', 'game', 'drama'], 'Sports': ['football', 'baseball', 'basketball', 'match', 'tournament'], 'Business': ['business', 'economy', 'stock', 'investment', 'market'], 'Health': ['health', 'medical', 'fitness', 'exercise', 'nutrition'], 'Science': ['science', 'research', 'experiment', 'discovery', 'theory'], 'Literature': ['novel', 'poem', 'literature', 'author', 'book'], 'Travel': ['travel', 'tourism', 'accommodation', 'flight', 'passport'] }; const custom = Config.settings.customKeywords || {}; for (const [cat, kws] of Object.entries(custom)) { standard[cat] = standard[cat] ? [...new Set([...standard[cat], ...kws])] : kws; } return standard; } return { classifyKeywords }; })(); // Evaluate Module const Evaluate = (() => { function evaluateContentQuality(text) { return text.split(/\s+/).length; } return { evaluateContentQuality }; })(); // Scheduler Module const Scheduler = (() => { let scheduleInterval = null; function scheduleCrawling() { clearInterval(scheduleInterval); const now = new Date(); const [startHour, startMinute] = Config.settings.schedule.startTime.split(':').map(Number); const startTime = new Date(now.getFullYear(), now.getMonth(), now.getDate(), startHour, startMinute, 0); if (startTime < now) startTime.setDate(startTime.getDate() + 1); const initialDelay = startTime - now; setTimeout(() => { Crawling.startCrawling(); scheduleInterval = setInterval(Crawling.startCrawling, Config.settings.schedule.interval); }, initialDelay); Logger.log(`Crawling schedule set. Start Time: ${startTime.toLocaleString()}, Interval: ${Config.settings.schedule.interval / 60000} minutes.`, 'info'); } function clearSchedule() { clearInterval(scheduleInterval); Logger.log('Crawling schedule has been cleared.', 'info'); } return { scheduleCrawling, clearSchedule }; })(); // Fetch Module // Already defined above // Robots Module // Already defined above // Extractor Module // Already defined above // Classifier Module // Already defined above // Evaluate Module // Already defined above // Download Module // Already defined above // Data Visualization Module // Already defined above // UI Module // Already defined above // Initialization function init() { UI.createUI(); UI.updateUI(); if (Config.settings.dataVisualization) DataVisualization.renderChart(); if (Config.settings.schedule.enabled) { Scheduler.scheduleCrawling(); } if (Config.settings.isCrawling) { Logger.log('Crawling has been resumed.', 'info'); Crawling.startCrawling(); } else { Logger.log('Crawler initialized.', 'info'); } } window.addEventListener('load', init);})(); 코드 리뷰나 코드 피드백 코드 평가 기타등등을 해줄 수 있을까??
- Assistant
확인해 보니, 당신이 작성한 "Advanced Web Crawler Enhanced Pro Optimized v6.2.1" 유저스크립트는 매우 포괄적이고 다양한 기능을 포함하고 있습니다. 아래에서 코드 전반에 대한 리뷰와 피드백을 제공하겠습니다.
1. 개요
이 유저스크립트는 다양한 검색 엔진에서 웹 크롤러를 실행하여 데이터를 수집하는 것을 목표로 합니다. 주요 기능은 다음과 같습니다:
- 구성 및 설정 관리: 다양한 크롤링 설정을 사용자 정의할 수 있는 설정 모듈 포함.
- 데이터 저장: IndexedDB를 사용하여 크롤링된 데이터, 로그, 방문한 URL 등을 저장.
- 로깅 및 UI: 사용자에게 실시간 로그와 크롤링 진행 상황을 보여주는 사용자 인터페이스 제공.
- 데이터 다운로드 및 시각화: 수집된 데이터를 CSV, JSON, XML, XLS 형식으로 다운로드하고, Chart.js를 사용하여 데이터 시각화.
- 스케줄링: 크롤링 작업을 일정에 따라 자동으로 실행할 수 있는 스케줄러 포함.
- 로봇 배제 준수 (robots.txt): 크롤링 전에 로봇 배제 표준을 확인하여 규정을 준수.
- CAPTCHA 감지 및 처리: CAPTCHA 발생 시 사용자에게 알림.
2. 강점
-
모듈화된 구조: 코드가 여러 개의 즉시 실행 함수(IIFE) 모듈로 나뉘어 있어, 각 기능별로 관리가 용이합니다. 예를 들어,
Config,DB,Logger,UI등으로 명확하게 분리되어 있습니다. -
풍부한 기능: 크롤링, 데이터 저장, 사용자 인터페이스, 데이터 시각화, 스케줄링 등 다양한 기능이 통합되어 있어, 사용자에게 강력한 도구를 제공합니다.
-
사용자 친화적인 UI: Tampermonkey를 위한 고급 사용자 인터페이스 제공. 레이아웃, 테마, 반응형 디자인 등이 잘 구현되어 있습니다.
-
데이터 저장의 효율성: IndexedDB를 활용하여 대용량 데이터를 효율적으로 저장하고 관리할 수 있습니다.
-
다양한 검색 엔진 지원: Google, Bing, Yahoo, DuckDuckGo 등 여러 검색 엔진을 지원하여, 광범위한 데이터 수집이 가능합니다.
-
접근성 및 반응형 디자인: 접근성을 고려한 설계와 다양한 화면 크기에 대응하는 반응형 UI는 사용자 경험을 향상시킵니다.
3. 개선이 필요한 부분 및 잠재적 이슈
가. 코드 구조 및 가독성
-
중복 코드: 여러 모듈에서 유사한 기능이나 로직이 반복됩니다. 예를 들어, 설정 저장 및 로딩 로직 등이 중복될 가능성이 있습니다. 이를 함수나 클래스 등으로 추출하여 코드 중복을 줄일 수 있습니다.
-
이벤트 핸들러의 복잡성: UI 모듈 내에서 많은 이벤트 핸들러가 등록되고 있는데, 이로 인해 코드가 복잡해지고 유지보수가 어려울 수 있습니다. 이벤트 위임(Event Delegation)을 활용하거나 이벤트 핸들러를 별도의 함수로 분리하는 것이 도움이 될 수 있습니다.
-
비동기 처리의 일관성 부족:
UI.updateLogs와 같은 함수는async로 정의되어 있지만, 호출 시await없이 사용됩니다. 이는 비동기 함수의 장점을 충분히 활용하지 못하게 할 수 있습니다.
나. 성능 및 최적화
-
동시 요청 제한 미준수:
Crawling모듈에서activeRequests를 관리하지만, 실제 HTTP 요청에 대한 속도 제어가 충분하지 않을 수 있습니다. 예를 들어, 허용된 동시 요청 수를 초과하지 않도록 보다 엄격한 제어가 필요할 수 있습니다. -
IndexedDB 성능: 대량의 데이터를 저장하거나 조회할 때 성능 저하가 발생할 수 있습니다. 인덱스 최적화나 배치 삽입 등의 방법을 고려할 수 있습니다.
다. 에러 핸들링
-
종합적인 에러 처리 부족: 일부 비동기 함수에서 에러 핸들링이 미흡할 수 있습니다. 예를 들어,
Crawling.crawlPage함수 내부에서 발생한 에러가 충분히 처리되지 않거나, 사용자에게 적절히 피드백되지 않을 수 있습니다. -
로봇 배제 표준 검사 실패 시 처리 부족:
Robots.checkRobotsTxt에서 실패 시 기본적으로 허용하지만, 이 경우 의도치 않은 크롤링이 발생할 수 있습니다. 보안상의 이유로 좀 더 엄격한 기본 정책을 고려할 수 있습니다.
라. 보안 및 프라이버시
-
프록시 사용 시 검증 부족: 사용자가 입력한 프록시 서버 주소에 대한 유효성 검사가 부족합니다. 이는 악의적인 프록시 서버로의 요청을 유발할 수 있어 보안 리스크가 있습니다.
-
데이터 다운로드 시 민감 정보 노출 가능성: 수집된 데이터에 민감한 정보가 포함될 경우, 이를 사용자에게 다운로드할 때 적절한 필터링이나 암호화가 필요할 수 있습니다.
마. 접근성 및 사용자 경험
-
ARIAL 속성 활용 부족: 일부 동적 요소에 대한 접근성 속성(
aria-*)이 부족합니다. 예를 들어, 모달 창을 열고 닫을 때aria-hidden속성을 적절히 관리하여 스크린 리더 사용자에게 상태 변화를 알릴 필요가 있습니다. -
키보드 내비게이션 개선: 현재
crawler-tab에tabindex가 설정되어 있으나, 키보드로의 미세한 내비게이션 및 단축키 지원이 더 강화될 수 있습니다.
바. 기타 이슈
-
검색 엔진 핸들러 부족: 코드에는 Google, Bing, DuckDuckGo 등 일부 검색 엔진의 핸들러만 구현되어 있습니다. 나머지 검색 엔진(Baidu, Yandex 등)에 대한 핸들러가 필요합니다.
-
CAPTCHA 처리의 한계: 현재 CAPTCHA를 감지하고 사용자에게 알리는 수준인데, 자동화된 처리는 아직 구현되지 않았습니다. CAPTCHA 우회는 법적 문제를 유발할 수 있으므로, 이를 명확히 고지하고 책임을 분명히 할 필요가 있습니다.
-
스케줄러의 재시작 로직 부족: 페이지가 새로 고침되거나, 스크립트가 재실행될 때 스케줄링 작업이 올바르게 유지되지 않을 수 있습니다. 이를 방지하기 위한 지속적인 스케줄 상태 유지 로직이 필요합니다.
4. 개선 제안
가. 코드 구조 및 가독성 개선
-
공통 기능 함수화: 설정 로딩, 저장, 사용자 입력 처리 등의 공통 기능을 별도의 유틸리티 함수나 클래스로 분리하여 코드 중복을 줄이고 가독성을 향상시킵니다.
-
이벤트 핸들러 분리: UI 모듈 내에서 이벤트 핸들러를 별도의 함수로 분리하여 코드의 조직화를 개선합니다.
-
비동기 함수 일관된 사용:
async함수는 항상await와 함께 사용하여 비동기 처리가 일관되게 이루어지도록 합니다.
나. 성능 및 최적화
-
요청 속도 제어 강화: 동시 요청 수를 엄격하게 제한하고, 큐 관리 로직을 개선하여 서버에 대한 부하를 줄입니다.
-
IndexedDB 최적화: 인덱스를 효과적으로 활용하거나, 데이터 삽입 시 배치 처리를 도입하여 저장 성능을 향상시킵니다.
다. 에러 핸들링 강화
-
전역 에러 핸들링: 예기치 않은 에러를 잡아내고, 사용자에게 명확히 알리는 전역 에러 핸들링 로직을 추가합니다.
-
robots.txt 검사 실패 시 기본 정책 강화: 기본적으로 크롤링을 금지하거나, 사용자에게 추가적인 확인을 요구하는 정책을 도입합니다.
라. 보안 및 프라이버시 강화
-
프록시 입력 검증: 사용자 입력 프록시 주소에 대한 엄격한 검증 로직을 추가하여, 악의적인 프록시 서버로의 요청을 방지합니다.
-
데이터 보호: 민감한 데이터가 포함될 수 있는 경우, 다운로드 시 데이터 필터링 옵션을 제공하거나, 암호화 기능을 고려합니다.
마. 접근성 및 사용자 경험 개선
-
ARIA 속성 완전 적용: 모든 동적 요소에 대해
aria-*속성을 적절히 적용하여 스크린 리더 사용자에게 명확한 정보를 제공합니다. -
키보드 내비게이션 강화: 모든 주요 기능에 대해 키보드 단축키를 정의하고, 키보드로의 접근성을 향상시킵니다.
-
사용자 피드백 개선: 크롤링 진행 상황, 오류 발생 시 사용자에게 보다 직관적이고 명확한 피드백을 제공하도록 UI를 개선합니다.
바. 기타 개선 사항
-
검색 엔진 핸들러 추가: Baidu, Yandex, Naver 등 다른 검색 엔진에 대한 핸들러를 구현하여 크롤링 범위를 확대합니다.
-
CAPTCHA 자동 처리 방지: 자동으로 CAPTCHA를 우회하려는 시도는 법적 문제를 유발할 수 있으므로, 현재 형태(사용자에게 알림)로 유지하고 명확한 책임 소재를 고지합니다.
-
스케줄러 상태 유지: 페이지가 새로 고침되거나 스크립트가 재실행될 때 스케줄링 상태를 유지하는 로직을 추가합니다. 예를 들어,
setInterval대신setTimeout과 같은 재귀적인 호출을 사용하여 더 안정적인 스케줄링을 구현할 수 있습니다.
5. 결론
작성하신 Advanced Web Crawler Enhanced Pro 유저스크립트는 다양한 기능과 모듈화를 통해 강력한 웹 크롤링 도구로서의 잠재력을 가지고 있습니다. 그러나 코드의 복잡성과 일부 기능의 부족, 에러 핸들링 및 보안 측면에서 몇 가지 개선이 필요합니다. 위에서 제시한 피드백과 개선 제안을 고려하여 코드를 보완한다면, 더욱 안정적이고 효율적인 크롤러로 발전시킬 수 있을 것입니다.
추가적으로, 크롤링 활동은 대상 웹사이트의 이용 약관 및 법적 제약을 준수해야 합니다. 로봇 배제 표준을 준수함으로써 합법적이고 윤리적인 크롤링을 수행하는 것이 중요합니다.
행운을 빕니다!