검색 엔진별 크롤링 로직의 일관성 유지 현재 extractSearchResults 함수 내에서 Google과 Bing에 대한 크롤링 로직이 구현되어 있으나,…
- User
검색 엔진별 크롤링 로직의 일관성 유지 현재 extractSearchResults 함수 내에서 Google과 Bing에 대한 크롤링 로직이 구현되어 있으나, 다른 검색 엔진에 대한 로직은 주석으로 처리되어 있습니다. 이는 스크립트의 완전성을 저해하며, 유지보수를 어렵게 만듭니다.
개선 방안:
검색 엔진별 핸들러 객체 , 에러 핸들링 개선 현재 크롤링 중 에러가 발생하면 로그를 남기지만, 에러의 원인을 보다 상세히 파악할 수 있는 방법이 부족합니다. 또한, 네트워크 요청 실패 시 모든 재시도 시도가 완료되었을 때의 처리가 부족합니다.
개선 방안:
상세 에러 로그: 에러 발생 시 더 많은 정보를 로그에 포함시킵니다. 재시도 로직 강화: 재시도 실패 시 크롤링을 중단하거나, 특정 동작을 추가로 수행합니다. , 비동기 처리의 효율성 향상 현재 crawlPage 함수는 페이지를 크롤링하면서 window.location.href를 변경하여 다음 페이지로 이동하고 있습니다. 이는 Tampermonkey 스크립트의 동작 방식과 충돌할 수 있으며, 스크립트의 연속성을 방해할 수 있습니다.
개선 방안:
브라우저 내비게이션 대신 AJAX 요청 사용: 페이지 전환 없이 배경에서 다음 페이지의 내용을 가져와 처리할 수 있습니다. 이를 통해 스크립트의 실행이 중단되지 않도록 합니다. 싱글 페이지 앱(SPA) 방식 고려: 만약 검색 엔진이 SPA 방식이라면, URL 변경 시 이벤트를 감지하여 크롤링을 계속할 수 있습니다. , 로딩 상태 표시: 대량의 데이터를 처리할 때 사용자에게 진행 상태를 명확히 알려주는 로딩 스피너나 애니메이션을 추가합니다. 에러 메시지 가시성 향상: 중요한 에러 메시지는 사용자에게 더 눈에 띄게 표시하여 즉각적으로 대응할 수 있도록 합니다. 설정 검증: 사용자 입력값에 대한 유효성을 검사하고, 잘못된 입력 시 경고를 표시하여 오류를 예방합니다. , 중복 URL 검출 최적화: 현재 visitedUrls는 배열로 관리되고 있는데, 대량의 URL을 처리할 경우 성능 저하가 발생할 수 있습니다. 이를 Set 자료구조로 변경하여 O(1)의 검색 시간을 확보할 수 있습니다. 이것들을 명령할꼐. 자 이제 이걸 바탕으로 밑에걸를 개선 해 일단 최대한 니가 시스템이 쓸 수 있는데 까지는 아무말없이 써봐. 그냥 코드만 써. ㅇㅇ 그리고 니가 써도 되는 용량이 최대가 될때 파트 1 으로 끝어 알겠지? // ==UserScript== // @name Advanced Web Crawler Enhanced Pro Optimized // @namespace http://tampermonkey.net/ // @version 6.0.3 // @description Optimized version of Advanced Web Crawler Enhanced Pro with enhanced UI controls and additional functionalities // @author Your Name // @match https://www.google.com/search* // @match https://www.bing.com/search* // @match https://search.yahoo.com/search* // @match https://duckduckgo.com/* // @match https://www.baidu.com/s* // @match https://www.yandex.com/search/* // @match https://search.naver.com/search* // @match https://search.daum.net/search* // @match https://www.ask.com/* // @match https://www.ecosia.org/search* // @match https://www.startpage.com/* // @match https://searx.me/* // @match https://www.dogpile.com/* // @match https://www.bing.com/images* // @grant GM_xmlhttpRequest // @grant GM_setValue // @grant GM_getValue // @grant GM_deleteValue // @connect * // @require https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js // @require https://cdn.jsdelivr.net/npm/chart.js // @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js // ==/UserScript==
(function() { 'use strict'; const defaultSettings = { delay: 2000, delayRandom: 500, maxPages: 3, userAgents: [ 'AdvancedWebCrawler/6.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)', 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15', 'Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15', ], userAgentIndex: 0, blacklist: [], whitelist: [], contentTypes: { include: ['text/html'], exclude: ['application/pdf', 'image/jpeg', 'image/png', 'video/mp4'] }, timeout: 8000, includeKeywords: [], excludeKeywords: [], language: [], minWordCount: 100, maxWordCount: 10000, maxDepth: 2, proxy: '', regexInclude: [], regexExclude: [], dataFields: { name: true, url: true, description: true, html: true, category: true, quality: true, language: true, wordCount: true, metaTags: false, headings: false }, schedule: { enabled: false, startTime: '00:00', interval: 3600000 }, searchEngines: { google: true, bing: true, yahoo: true, duckduckgo: true, baidu: true, yandex: true, naver: true, daum: true, ask: true, ecosia: true, startpage: true, searx: true, dogpile: true, bingimages: true }, retryAttempts: 3, retryDelay: 2000, respectRobots: true, maxConcurrentRequests: 3, dataVisualization: true, exportFormats: { csv: true, json: true, xml: true, xls: true }, advancedLogging: true, enableCaptchas: false }; const settings = Object.assign({}, defaultSettings, { delay: parseInt(GM_getValue('delay', defaultSettings.delay), 10), delayRandom: parseInt(GM_getValue('delayRandom', defaultSettings.delayRandom), 10), maxPages: parseInt(GM_getValue('maxPages', defaultSettings.maxPages), 10), userAgents: GM_getValue('userAgents', defaultSettings.userAgents), userAgentIndex: parseInt(GM_getValue('userAgentIndex', defaultSettings.userAgentIndex), 10), blacklist: GM_getValue('blacklist', defaultSettings.blacklist), whitelist: GM_getValue('whitelist', defaultSettings.whitelist), contentTypes: GM_getValue('contentTypes', defaultSettings.contentTypes), timeout: parseInt(GM_getValue('timeout', defaultSettings.timeout), 10), includeKeywords: GM_getValue('includeKeywords', defaultSettings.includeKeywords), excludeKeywords: GM_getValue('excludeKeywords', defaultSettings.excludeKeywords), regexInclude: GM_getValue('regexInclude', defaultSettings.regexInclude), regexExclude: GM_getValue('regexExclude', defaultSettings.regexExclude), language: GM_getValue('language', defaultSettings.language), minWordCount: parseInt(GM_getValue('minWordCount', defaultSettings.minWordCount), 10), maxWordCount: parseInt(GM_getValue('maxWordCount', defaultSettings.maxWordCount), 10), maxDepth: parseInt(GM_getValue('maxDepth', defaultSettings.maxDepth), 10), proxy: GM_getValue('proxy', defaultSettings.proxy), dataFields: GM_getValue('dataFields', defaultSettings.dataFields), schedule: GM_getValue('schedule', defaultSettings.schedule), searchEngines: GM_getValue('searchEngines', defaultSettings.searchEngines), retryAttempts: parseInt(GM_getValue('retryAttempts', defaultSettings.retryAttempts), 10), retryDelay: parseInt(GM_getValue('retryDelay', defaultSettings.retryDelay), 10), respectRobots: GM_getValue('respectRobots', defaultSettings.respectRobots), maxConcurrentRequests: parseInt(GM_getValue('maxConcurrentRequests', defaultSettings.maxConcurrentRequests), 10), dataVisualization: GM_getValue('dataVisualization', defaultSettings.dataVisualization), exportFormats: GM_getValue('exportFormats', defaultSettings.exportFormats), advancedLogging: GM_getValue('advancedLogging', defaultSettings.advancedLogging), enableCaptchas: GM_getValue('enableCaptchas', defaultSettings.enableCaptchas), customKeywords: GM_getValue('customKeywords', {}) }); let visitedUrls = GM_getValue('visitedUrls', []); let logs = GM_getValue('logs', []); let data = GM_getValue('data', []); let isCrawling = GM_getValue('isCrawling', false); let currentPage = GM_getValue('currentPage', 1); let totalPages = GM_getValue('totalPages', settings.maxPages); let progress = GM_getValue('progress', 0); let currentDepth = GM_getValue('currentDepth', 0); let activeRequests = 0; let requestQueue = []; let previousCrawls = GM_getValue('previousCrawls', []); function createUI() { const style =
#crawler-container { position: fixed; top: 20px; right: 20px; width: 400px; height: 500px; background: #fff; border: 1px solid #ccc; z-index: 10000; font-family: Arial, sans-serif; box-shadow: 0 4px 8px rgba(0,0,0,0.2); overflow: auto; resize: both; border-radius: 8px; min-width: 300px; min-height: 300px; } #crawler-header { display: flex; justify-content: space-between; align-items: center; background: #f1f1f1; border-bottom: 1px solid #ccc; padding: 5px 10px; cursor: move; } #crawler-minimize { background: none; border: none; font-size: 18px; cursor: pointer; } #crawler-tabs { display: flex; background: #f1f1f1; border-bottom: 1px solid #ccc; } .crawler-tab { flex: 1; padding: 10px; text-align: center; cursor: pointer; font-weight: bold; } .crawler-tab.active { background: #ddd; } #crawler-content { padding: 10px; overflow: auto; height: calc(100% - 50px); } table { width: 100%; border-collapse: collapse; } th, td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; } th { background-color: #f2f2f2; cursor: pointer; } tr:hover { background-color: #f5f5f5; } .progress-bar { width: 100%; background-color: #f3f3f3; border: 1px solid #ccc; border-radius: 5px; overflow: hidden; margin-bottom: 10px; } .progress-bar-fill { height: 20px; background-color: #4caf50; width: 0%; transition: width 0.2s; } .modal { display: none; position: fixed; z-index: 10001; left: 0; top: 0; width: 100%; height: 100%; overflow: auto; background-color: rgba(0,0,0,0.4); } .modal-content { background-color: #fff; margin: 5% auto; padding: 20px; border: 1px solid #888; width: 80%; max-height: 80%; overflow-y: auto; border-radius: 8px; } .close { color: #aaa; float: right; font-size: 28px; font-weight: bold; cursor: pointer; } .control-buttons { margin: 10px 0; } .control-buttons button { padding: 8px 12px; margin-right: 8px; cursor: pointer; border: none; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px; } .control-buttons button:disabled { background-color: #a5d6a7; cursor: not-allowed; } .status-text { margin-top: 10px; font-weight: bold; font-size: 14px; } .settings-section { margin-bottom: 15px; } .settings-section label { display: block; margin-bottom: 5px; font-weight: bold; } .settings-section input[type="text"], .settings-section input[type="number"], .settings-section textarea, .settings-section select { width: 100%; padding: 5px; box-sizing: border-box; border: 1px solid #ccc; border-radius: 4px; font-size: 14px; } .settings-section .checkbox-group { display: flex; flex-wrap: wrap; } .settings-section .checkbox-group label { margin-right: 10px; font-size: 14px; } .settings-section .radio-group { display: flex; flex-wrap: wrap; } .settings-section .radio-group label { margin-right: 10px; font-size: 14px; } .tooltip { position: relative; display: inline-block; cursor: help; font-size: 14px; } .tooltip .tooltiptext { visibility: hidden; width: 220px; background-color: #555; color: #fff; text-align: center; border-radius: 6px; padding: 5px; position: absolute; z-index: 1; bottom: 125%; left: 50%; margin-left: -110px; opacity: 0; transition: opacity 0.3s; } .tooltip:hover .tooltiptext { visibility: visible; opacity: 1; } #crawler-container::-webkit-resizer { background: #f1f1f1; border: none; } #data-visualization { margin-top: 20px; } #data-chart { width: 100%; height: 300px; } #modal-html-content { white-space: pre-wrap; word-wrap: break-word; background-color: #f9f9f9; padding: 10px; border-radius: 4px; } #crawler-tabs .crawler-tab:last-child { flex: 1; } #previous-crawls { margin-top: 20px; } #previous-crawls table { width: 100%; } #previous-crawls th, #previous-crawls td { padding: 5px; } #reset-data { background-color: #f44336; }; {style}); const container = $(📜 로그 📊 데이터 ⚙️ 설정 📁 기록 ─ 진행률: {currentPage}/{isCrawling ? 'disabled' : ''}>▶ 시작 <button id="stop-crawling" title="크롤링을 중지합니다." {settings.exportFormats.csv ?<button id="download-csv" title="CSV 다운로드">📥 CSV</button>: ''} {settings.exportFormats.xml ?<button id="download-xml" title="XML 다운로드">📥 XML</button>: ''} {settings.exportFormats.xls ? `<button id="download-xls" title="XLS 다운로드">📥 XLS</button>` : ''} <button id="reset-data" title="모든 크롤링 데이터를 초기화합니다.">🗑️ 초기화</button> </div> <div id="log-messages" style="height: 200px; overflow-y: auto; background: #f9f9f9; padding: 10px; border: 1px solid #ccc; border-radius: 4px;">로그 내용이 여기에 표시됩니다.</div> </div> <div id="tab-data" style="display: {currentTab() === 'data' ? 'block' : 'none'};"> {settings.dataFields.url ? '주소' : ''} {settings.dataFields.html ? '📄 HTML 내용' : ''} {settings.dataFields.quality ? '품질점수' : ''} {settings.dataFields.wordCount ? '단어 수' : ''} {settings.dataFields.headings ? '헤딩' : ''} {currentTab() === 'settings' ? 'block' : 'none'};"> 설정 크롤링 딜레이 (ms): ℹ️ 기본 딜레이과 딜레이 랜덤 값을 조합하여 요청 사이의 딜레이를 설정합니다. 딜레이 랜덤 범위 (ms): ℹ️ 딜레이에 무작위 요소를 추가하여 보다 자연스러운 요청을 합니다. 최대 크롤링 페이지 수: ℹ️ 크롤링할 최대 페이지 수를 설정하세요. 사용자 에이전트(User-Agent): ℹ️ 크롤링에 사용할 사용자 에이전트를 선택하거나 추가할 수 있습니다. {settings.userAgents.map((ua, idx) => `<option value="{idx}" {ua}).join('')} </select> <button id="add-user-agent" style="margin-top: 5px; padding: 5px 10px; background-color: #2196F3; color: white; border-radius: 4px; font-size: 12px;">추가</button> <button id="remove-user-agent" style="margin-top: 5px; padding: 5px 10px; background-color: #f44336; color: white; border-radius: 4px; font-size: 12px;">제거</button> </div> <div class="settings-section"> <label>콘텐츠 유형:</label> <div> <strong>포함할 유형:</strong> <textarea id="content-include-input" rows="3">${settings.contentTypes.include.join(', ')}</textarea> </div> <div> <strong>제외할 유형:</strong> <textarea id="content-exclude-input" rows="3">${settings.contentTypes.exclude.join(', ')}</textarea> </div> </div> <div class="settings-section"> <label for="timeout-input">크롤링 타임아웃 (ms): <span class="tooltip">ℹ️ <span class="tooltiptext">요청이 완료되기까지 기다릴 최대 시간을 밀리초 단위로 설정합니다.</span> </span> </label> <input type="number" id="timeout-input" value="${settings.timeout}" min="1000" step="1000"> </div> <div class="settings-section"> <label for="include-keywords-input">포함할 키워드: <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링할 페이지에 반드시 포함되어야 하는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="include-keywords-input" rows="3">${settings.includeKeywords.join('\n')}</textarea> </div> <div class="settings-section"> <label for="exclude-keywords-input">제외할 키워드: <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링할 페이지에 포함되면 안 되는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="exclude-keywords-input" rows="3">${settings.excludeKeywords.join('\n')}</textarea> </div> <div class="settings-section"> <label for="regex-include-input">포함할 키워드 (정규 표현식): <span class="tooltip">ℹ️ <span class="tooltiptext">포함할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="regex-include-input" rows="3">${settings.regexInclude.join('\n')}</textarea> </div> <div class="settings-section"> <label for="regex-exclude-input">제외할 키워드 (정규 표현식): <span class="tooltip">ℹ️ <span class="tooltiptext">제외할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="regex-exclude-input" rows="3">${settings.regexExclude.join('\n')}</textarea> </div> <div class="settings-section"> <label for="language-select">언어 필터링:</label> <select id="language-select" multiple size="10"> ${[ { value: 'en', text: '영어 (en)' }, { value: 'ko', text: '한국어 (ko)' }, { value: 'zh', text: '중국어 (zh)' }, { value: 'ja', text: '일본어 (ja)' }, { value: 'ru', text: '러시아어 (ru)' }, { value: 'es', text: '스페인어 (es)' }, { value: 'fr', text: '프랑스어 (fr)' }, { value: 'de', text: '독일어 (de)' }, { value: 'it', text: '이탈리아어 (it)' }, { value: 'pt', text: '포르투갈어 (pt)' }, { value: 'ar', text: '아랍어 (ar)' }, { value: 'hi', text: '힌디어 (hi)' }, { value: 'bn', text: '벵골어 (bn)' }, { value: 'pa', text: '펀자브어 (pa)' }, { value: 'nl', text: '네덜란드어 (nl)' }, { value: 'sv', text: '스웨덴어 (sv)' }, { value: 'tr', text: '터키어 (tr)' }, { value: 'vi', text: '베트남어 (vi)' }, { value: 'pl', text: '폴란드어 (pl)' }, { value: 'th', text: '태국어 (th)' } ].map(lang =><option value="{settings.language.includes(lang.value) ? 'selected' : ''}>{lang.text}</option>`).join('')} </select> </div> <div class="settings-section"> <label for="min-word-count-input">최소 단어 수: <span class="tooltip">ℹ️ <span class="tooltiptext">수집할 페이지의 최소 단어 수를 설정하세요.</span> </span> </label> <input type="number" id="min-word-count-input" value="{settings.minWordCount}" min="10" step="10"> 최대 단어 수: ℹ️ 수집할 페이지의 최대 단어 수를 설정하세요. 최대 크롤링 깊이: ℹ️ 링크를 추적할 최대 깊이를 설정하세요. 프록시 서버: ℹ️ 프록시 서버 주소를 입력하세요. 예: http://proxyserver:port 수집할 데이터 필드: {Object.keys(settings.dataFields).map(field => `<label><input type="checkbox" id="field-{field}" {getFieldName(field)}).join('')} </div> </div> <div class="settings-section"> <label for="retry-attempts-input">재시도 횟수: <span class="tooltip">ℹ️ <span class="tooltiptext">요청 실패 시 재시도할 횟수를 설정하세요.</span> </span> </label> <input type="number" id="retry-attempts-input" value="${settings.retryAttempts}" min="0" step="1"> <label for="retry-delay-input">재시도 딜레이 (ms): <span class="tooltip">ℹ️ <span class="tooltiptext">재시도 시 대기할 시간을 밀리초 단위로 설정하세요.</span> </span> </label> <input type="number" id="retry-delay-input" value="${settings.retryDelay}" min="1000" step="500"> </div> <div class="settings-section"> <label>크롤링 스케줄링:</label> <div class="checkbox-group"> <label><input type="checkbox" id="schedule-enabled" ${settings.schedule.enabled ? 'checked' : ''}> 스케줄링 활성화</label> </div> <div id="schedule-options" style="display: ${settings.schedule.enabled ? 'block' : 'none'}; margin-top: 10px;"> <label for="start-time-input">시작 시간 (HH:MM): <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링을 시작할 시간을 설정하세요.</span> </span> </label> <input type="time" id="start-time-input" value="${settings.schedule.startTime}"> <label for="interval-input">크롤링 간격 (분): <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링 작업 간의 간격을 분 단위로 설정하세요.</span> </span> </label> <input type="number" id="interval-input" value="${settings.schedule.interval / 60000}" min="1" step="1"> </div> </div> <div class="settings-section"> <label>크롤링할 검색 엔진 선택:</label> <div class="checkbox-group"> ${Object.keys(settings.searchEngines).map(engine =><input type="checkbox" id="engine-{settings.searchEngines[engine] ? 'checked' : ''}> {capitalize(engine)}</label>`).join('')} </div> </div> <div class="settings-section"> <label for="blacklist-input">블랙리스트 도메인: <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링에서 제외할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="blacklist-input" rows="5">{settings.blacklist.join('\n')} 화이트리스트 도메인: ℹ️ 크롤링을 허용할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다. {settings.respectRobots ? 'checked' : ''}> 동시 요청 최대 수: ℹ️ 한 번에 요청할 수 있는 최대 동시 요청 수를 설정하세요. 데이터 시각화 활성화: ℹ️ 수집된 데이터를 시각화하여 그래프로 표시합니다. <input type="checkbox" id="data-visualization-checkbox" {settings.enableCaptchas ? 'checked' : ''}> 데이터 내보내기 형식: {Object.keys(settings.exportFormats).map(format => `<label><input type="checkbox" id="export-{format}" {format.toUpperCase()}).join('')} </div> </div> <div class="settings-section"> <label for="advanced-logging-checkbox">고급 로깅 활성화: <span class="tooltip">ℹ️ <span class="tooltiptext">추가적인 로깅 정보를 활성화합니다.</span> </span> </label> <input type="checkbox" id="advanced-logging-checkbox" ${settings.advancedLogging ? 'checked' : ''}> </div> <div class="settings-section"> <label for="custom-keywords-input">사용자 지정 키워드 분류: <span class="tooltip">ℹ️ <span class="tooltiptext">사용자 지정 키워드를 추가하여 데이터 분류를 향상시킵니다. 형식: 카테고리:키워드1,키워드2,...</span> </span> </label> <textarea id="custom-keywords-input" rows="5">${formatCustomKeywords(settings.customKeywords)}</textarea> </div> <button id="save-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">저장</button> <button id="reset-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">초기화</button> </div> <div id="tab-previous-crawls" style="display: ${currentTab() === 'previous-crawls' ? 'block' : 'none'};"> <h3>이전 크롤링 기록</h3> <button id="clear-previous-crawls" style="padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px; margin-bottom: 10px;">기록 초기화</button> <table> <thead> <tr> <th>날짜</th> <th>크롤링 수</th> <th>상태</th> </tr> </thead> <tbody id="previous-crawls-body"> ${previousCrawls.map(crawl =>{crawl.count} {crawl.status}</td> </tr> `).join('')} </tbody> </table> </div> </div> <div id="html-modal" class="modal"> <div class="modal-content"> <span class="close">×</span> <pre id="modal-html-content"></pre> </div> </div> </div> `); ('body').append(container); ('#crawler-minimize').on('click', function() { ('#crawler-content').toggle(); const isHidden = ('#crawler-content').is(':hidden'); (this).text(isHidden ? '☐' : '─'); log(크롤러 창이 ${isHidden ? '최소화' : '확대'}되었습니다., 'info'); }); ('.crawler-tab').on('click', function() { const tab = (this).data('tab'); (this).addClass('active'); ('#crawler-content > div').hide(); (#tab-${tab}).show(); saveCurrentTab(tab); }); (document).on('keydown', function(e) { if (e.ctrlKey) { const actions = { '1': 'logs', '2': 'data', '3': 'settings', '4': 'previous-crawls', 's': 'save', 'r': 'reset', 'p': 'start', 'q': 'stop', 'c': 'csv', 'j': 'json', 'x': 'xml', 'l': 'xls' }; const action = actions[e.key]; if (action) { e.preventDefault(); switch(action) { case 'logs': ('.crawler-tab[data-tab="logs"]').click(); break; case 'data': ('.crawler-tab[data-tab="settings"]').click(); break; case 'previous-crawls': ('.crawler-tab[data-tab="previous-crawls"]').click(); break; case 'save': saveSettings(); break; case 'reset': resetSettings(); break; case 'start': if (!isCrawling) startCrawling(); break; case 'stop': if (isCrawling) stopCrawling(); break; case 'csv': settings.exportFormats.csv && downloadCSV(); break; case 'json': settings.exportFormats.json && downloadJSON(); break; case 'xml': settings.exportFormats.xml && downloadXML(); break; case 'xls': settings.exportFormats.xls && downloadXLS(); break; } } } }); ('#save-settings').on('click', saveSettings); ('#reset-settings').on('click', resetSettings); ('#add-user-agent').on('click', function() { const ua = prompt("새 사용자 에이전트를 입력하세요:"); if (ua) { settings.userAgents.push(ua); GM_setValue('userAgents', settings.userAgents); updateUserAgentSelect(); log('새 사용자 에이전트가 추가되었습니다.', 'success'); } }); ('#remove-user-agent').on('click', function() { const selected = ('#user-agent-select').val(); if (selected && selected.length) { settings.userAgents = settings.userAgents.filter((_, idx) => !selected.includes(idx.toString())); GM_setValue('userAgents', settings.userAgents); settings.userAgentIndex = 0; GM_setValue('userAgentIndex', settings.userAgentIndex); updateUserAgentSelect(); log('선택한 사용자 에이전트가 제거되었습니다.', 'success'); } else alert('제거할 사용자 에이전트를 선택하세요.'); }); ('#download-csv').on('click', downloadCSV); ('#download-json').on('click', downloadJSON); ('#download-xml').on('click', downloadXML); ('#download-xls').on('click', downloadXLS); ('#search-filter').on('input', function() { filterTable((this).val()); }); ('#schedule-enabled').on('change', function() { ('#schedule-options').toggle((this).is(':checked')); }); ('#data-visualization-checkbox').on('change', function() { settings.dataVisualization = ('#data-visualization').toggle(settings.dataVisualization); log('데이터 시각화 설정이 변경되었습니다.', 'success'); if (settings.dataVisualization) renderChart(); }); ('#enable-captchas-checkbox').on('change', function() { settings.enableCaptchas = (this).is(':checked'); GM_setValue('enableCaptchas', settings.enableCaptchas); log('CAPTCHA 처리 설정이 변경되었습니다.', 'success'); }); ('.view-html').on('click', function() { const html = decodeURIComponent((this).data('html')); ('#modal-html-content').text(html); ('#html-modal').show(); }); ('.close').on('click', function() { ('#html-modal').hide(); }); (window).on('click', function(e) { if (e.target.id === 'html-modal') ('#html-modal').hide(); }); ('#start-crawling').on('click', startCrawling); ('#stop-crawling').on('click', stopCrawling); ('#download-csv').on('click', downloadCSV); ('#download-json').on('click', downloadJSON); ('#download-xml').on('click', downloadXML); ('#download-xls').on('click', downloadXLS); ('#clear-previous-crawls').on('click', function() { if (confirm("이전 크롤링 기록을 모두 초기화하시겠습니까?")) { previousCrawls = []; GM_setValue('previousCrawls', previousCrawls); ('#previous-crawls-body').empty(); log('이전 크롤링 기록이 초기화되었습니다.', 'success'); } }); ('#reset-data').on('click', function() { if (confirm("모든 크롤링 데이터를 초기화하시겠습니까?")) { data = []; visitedUrls = []; GM_setValue('data', data); GM_setValue('visitedUrls', visitedUrls); updateDataUI(); log('크롤링 데이터가 초기화되었습니다.', 'success'); } }); updateLogsUI(); updateDataUI(); if (settings.dataVisualization) renderChart(); updatePreviousCrawlsUI(); makeDraggable(); } function currentTab() { return GM_getValue('currentTab', 'logs'); } function saveCurrentTab(tab) { GM_setValue('currentTab', tab); } function updateUserAgentSelect() { ('#user-agent-select').empty(); settings.userAgents.forEach((ua, idx) => { ('#user-agent-select').append(`<option value="{idx}" {ua}); }); } function getCustomKeywordsText() { const ck = settings.customKeywords || {}; return Object.entries(ck).map(([cat, kws]) =>{kws.join(',')}).join('\n'); } function parseCustomKeywords(text) { const lines = text.split('\n').map(l => l.trim()).filter(l => l); const ck = {}; lines.forEach(line => { const [cat, kws] = line.split(':'); if (cat && kws) ck[cat.trim()] = kws.split(',').map(kw => kw.trim()).filter(kw => kw); }); return ck; } function combineKeywords() { const standard = { '교육': ['학교', '교육', '학생', '교수', '강의'], '기술': ['기술', '소프트웨어', '하드웨어', '프로그래밍', '개발'], '뉴스': ['뉴스', '속보', '기사', '보도'], '엔터테인먼트': ['영화', '음악', '연예', '게임', '드라마'], '스포츠': ['축구', '야구', '농구', '경기', '토너먼트'], '비즈니스': ['비즈니스', '경제', '주식', '투자', '시장'], '건강': ['건강', '의료', '헬스', '운동', '영양'], '과학': ['과학', '연구', '실험', '발견', '이론'], '문학': ['소설', '시', '문학', '작가', '책'], '여행': ['여행', '관광', '숙박', '비행기', '여권'] }; const custom = settings.customKeywords || {}; for (const [cat, kws] of Object.entries(custom)) { standard[cat] = standard[cat] ? [...new Set([...standard[cat], ...kws])] : kws; } return standard; } function log(message, type = 'info') { const timestamp = new Date().toLocaleString(); logs.push({ timestamp, message, type }); GM_setValue('logs', logs); updateLogsUI(); if (settings.advancedLogging) { const styles = { 'info': 'color: blue;', 'warning': 'color: orange;', 'error': 'color: red;', 'success': 'color: green; font-weight: bold;' }; console.log(%c[{type.toUpperCase()}] {message}`, styles[type] || ''); } } function updateLogsUI() { ('#log-messages').html(logs.map(l =><div>[${l.timestamp}] [${l.type.toUpperCase()}] ${l.message}</div>).join('')); {progress.toFixed(2)}%); $('.status-text').text(진행률: {currentPage}/{totalPages} 페이지)`); } function updateDataUI() { const filter = ('#search-filter').val().toLowerCase(); const filtered = filter ? data.filter(item => Object.keys(settings.dataFields).some(field => settings.dataFields[field] && String(item[field]).toLowerCase().includes(filter))) : data; const rows = filtered.map(item =><tr> ${settings.dataFields.name ?{escapeHTML(item.name)}</td>` : ''} {settings.dataFields.url ?<td><a href="${escapeHTML(item.url)}" target="_blank">${escapeHTML(item.url)}</a></td>: ''} {settings.dataFields.description ? `<td>{escapeHTML(item.description)}: ''} ${settings.dataFields.html ?📄 보기: ''} ${settings.dataFields.category ?{escapeHTML(item.category)}</td>` : ''} {settings.dataFields.quality ?<td>${escapeHTML(item.quality.toString())}</td>: ''} {settings.dataFields.language ? `<td>{escapeHTML(item.language)}: ''} ${settings.dataFields.wordCount ?{escapeHTML(item.wordCount.toString())}</td>` : ''} {settings.dataFields.metaTags ?<td>${escapeHTML(JSON.stringify(item.metaTags))}</td>: ''} {settings.dataFields.headings ? `<td>{escapeHTML(item.headings.join(', '))}: ''} </tr>).join(''); ('#data-table-body').html(rows); ('.view-html').off('click').on('click', function() { ('#modal-html-content').text(decodeURIComponent((this).data('html'))); ('#html-modal').show(); }); if (settings.dataVisualization) renderChart(); } function renderChart() { if (!document.getElementById('data-chart').getContext) return; const ctx = document.getElementById('data-chart').getContext('2d'); if (window.dataChartInstance) window.dataChartInstance.destroy(); const categories = {}; data.forEach(item => { const cat = item.category || '기타'; categories[cat] = (categories[cat] || 0) + 1; }); const chartData = { labels: Object.keys(categories), datasets: [{ label: '카테고리별 데이터 수', data: Object.values(categories), backgroundColor: generateColors(Object.keys(categories).length) }] }; window.dataChartInstance = new Chart(ctx, { type: 'pie', data: chartData, options: { responsive: true } }); } function generateColors(count) { return Array.from({length: count}, (_, i) => `hsl({i * (360 / count)}, 70%, 50%)); } function escapeHTML(str) { return String(str).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'"); } function downloadCSV() { if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; } const headers = Object.keys(settings.dataFields).filter(f => settings.dataFields[f]).map(f => getFieldName(f)); const csv = [headers.join(",")].concat(data.map(item => Object.keys(settings.dataFields).filter(f => settings.dataFields[f]).map(f =>"{encodeURIComponent(csv)}; link.download = "data.csv"; document.body.appendChild(link); link.click(); document.body.removeChild(link); log('CSV 파일이 다운로드되었습니다.', 'success'); } function downloadJSON() { if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; } const json = JSON.stringify(data, null, 2); const blob = new Blob([json], { type: "application/json" }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); link.download = "data.json"; document.body.appendChild(link); link.click(); document.body.removeChild(link); log('JSON 파일이 다운로드되었습니다.', 'success'); } function downloadXML() { if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; } let xml = '<?xml version="1.0" encoding="UTF-8"?><data>'; data.forEach(item => { xml += '<entry>'; if (settings.dataFields.name) xml +={escapeXML(item.url || '')}; if (settings.dataFields.description) xml +={item.html || ''}]]>; if (settings.dataFields.category) xml +={escapeXML(item.quality)}; if (settings.dataFields.language) xml +={escapeXML(item.wordCount)}; if (settings.dataFields.metaTags) xml +={escapeXML(item.headings.join('; '))}; xml += '</entry>'; }); xml += '</data>'; const blob = new Blob([xml], { type: "application/xml" }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); link.download = "data.xml"; document.body.appendChild(link); link.click(); document.body.removeChild(link); log('XML 파일이 다운로드되었습니다.', 'success'); } function downloadXLS() { if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; } const sheet = XLSX.utils.json_to_sheet(data.map(item => { const entry = {}; if (settings.dataFields.name) entry['사이트 이름'] = item.name; if (settings.dataFields.url) entry['주소'] = item.url; if (settings.dataFields.description) entry['설명'] = item.description; if (settings.dataFields.html) entry['📄 HTML 내용'] = item.html; if (settings.dataFields.category) entry['분류'] = item.category; if (settings.dataFields.quality) entry['품질점수'] = item.quality; if (settings.dataFields.language) entry['언어'] = item.language; if (settings.dataFields.wordCount) entry['단어 수'] = item.wordCount; if (settings.dataFields.metaTags) entry['메타 태그'] = JSON.stringify(item.metaTags); if (settings.dataFields.headings) entry['헤딩'] = item.headings.join('; '); return entry; })); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, sheet, "Data"); XLSX.writeFile(wb, "data.xlsx"); log('XLS 파일이 다운로드되었습니다.', 'success'); } function escapeCSV(str) { return String(str).replace(/"/g, '""'); } function filterTable(keyword) { $('#data-table-body tr').each(function() { $(this).toggle($(this).text().toLowerCase().includes(keyword.toLowerCase())); }); } function saveSettings() { settings.delay = parseInt($('#delay-input').val(), 10) || defaultSettings.delay; settings.delayRandom = parseInt($('#delay-random-input').val(), 10) || defaultSettings.delayRandom; settings.maxPages = parseInt($('#max-pages-input').val(), 10) || defaultSettings.maxPages; const selectedUA = parseInt($('#user-agent-select').val(), 10); settings.userAgentIndex = isNaN(selectedUA) ? 0 : selectedUA; settings.contentTypes.include = $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(ct => ct) || defaultSettings.contentTypes.include; settings.contentTypes.exclude = $('#content-exclude-input').val().split(',').map(ct => ct.trim()).filter(ct => ct) || defaultSettings.contentTypes.exclude; settings.timeout = parseInt($('#timeout-input').val(), 10) || defaultSettings.timeout; settings.includeKeywords = $('#include-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw) || defaultSettings.includeKeywords; settings.excludeKeywords = $('#exclude-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw) || defaultSettings.excludeKeywords; settings.regexInclude = $('#regex-include-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx) || defaultSettings.regexInclude; settings.regexExclude = $('#regex-exclude-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx) || defaultSettings.regexExclude; settings.language = $('#language-select').val() || []; settings.minWordCount = parseInt($('#min-word-count-input').val(), 10) || defaultSettings.minWordCount; settings.maxWordCount = parseInt($('#max-word-count-input').val(), 10) || defaultSettings.maxWordCount; settings.maxDepth = parseInt($('#max-depth-input').val(), 10) || defaultSettings.maxDepth; settings.proxy = $('#proxy-input').val().trim() || defaultSettings.proxy; ['name','url','description','html','category','quality','language','wordCount','metaTags','headings'].forEach(field => { settings.dataFields[field] = $(#field-{field}`).is(':checked'); }); settings.retryAttempts = parseInt(('#retry-attempts-input').val(), 10) || defaultSettings.retryAttempts; settings.retryDelay = parseInt(('#retry-delay-input').val(), 10) || defaultSettings.retryDelay; settings.schedule.enabled = ('#schedule-enabled').is(':checked'); settings.schedule.startTime = ('#start-time-input').val() || defaultSettings.schedule.startTime; settings.schedule.interval = parseInt(('#interval-input').val(), 10) ? parseInt(('#interval-input').val(), 10) * 60000 : defaultSettings.schedule.interval; Object.keys(settings.searchEngines).forEach(engine => { settings.searchEngines[engine] = (#engine-${engine}).is(':checked'); }); settings.blacklist = ('#blacklist-input').val().split('\n').map(d => d.trim()).filter(d => d) || defaultSettings.blacklist; settings.whitelist = ('#whitelist-input').val().split('\n').map(d => d.trim()).filter(d => d) || defaultSettings.whitelist; settings.respectRobots = ('#respect-robots-checkbox').is(':checked'); settings.maxConcurrentRequests = parseInt(('#max-concurrent-requests-input').val(), 10) || defaultSettings.maxConcurrentRequests; settings.dataVisualization = ('#data-visualization-checkbox').is(':checked'); settings.exportFormats.csv = ('#export-csv').is(':checked'); settings.exportFormats.json = ('#export-json').is(':checked'); settings.exportFormats.xml = ('#export-xml').is(':checked'); settings.exportFormats.xls = ('#export-xls').is(':checked'); settings.advancedLogging = ('#advanced-logging-checkbox').is(':checked'); settings.enableCaptchas = ('#enable-captchas-checkbox').is(':checked'); const customKeywords = parseCustomKeywords(('#custom-keywords-input').val()); settings.customKeywords = customKeywords; GM_setValue('delay', settings.delay); GM_setValue('delayRandom', settings.delayRandom); GM_setValue('maxPages', settings.maxPages); GM_setValue('userAgentIndex', settings.userAgentIndex); GM_setValue('contentTypes', settings.contentTypes); GM_setValue('timeout', settings.timeout); GM_setValue('includeKeywords', settings.includeKeywords); GM_setValue('excludeKeywords', settings.excludeKeywords); GM_setValue('regexInclude', settings.regexInclude); GM_setValue('regexExclude', settings.regexExclude); GM_setValue('language', settings.language); GM_setValue('minWordCount', settings.minWordCount); GM_setValue('maxWordCount', settings.maxWordCount); GM_setValue('maxDepth', settings.maxDepth); GM_setValue('proxy', settings.proxy); GM_setValue('dataFields', settings.dataFields); GM_setValue('retryAttempts', settings.retryAttempts); GM_setValue('retryDelay', settings.retryDelay); GM_setValue('schedule', settings.schedule); GM_setValue('searchEngines', settings.searchEngines); GM_setValue('blacklist', settings.blacklist); GM_setValue('whitelist', settings.whitelist); GM_setValue('respectRobots', settings.respectRobots); GM_setValue('maxConcurrentRequests', settings.maxConcurrentRequests); GM_setValue('dataVisualization', settings.dataVisualization); GM_setValue('exportFormats', settings.exportFormats); GM_setValue('advancedLogging', settings.advancedLogging); GM_setValue('enableCaptchas', settings.enableCaptchas); GM_setValue('customKeywords', settings.customKeywords); totalPages = settings.maxPages; GM_setValue('totalPages', totalPages); log('설정이 저장되었습니다.', 'success'); if (settings.schedule.enabled) scheduleCrawling(); updateDataUI(); if (settings.dataVisualization) renderChart(); } function resetSettings() { if (confirm("기본 설정으로 초기화하시겠습니까?")) { Object.keys(defaultSettings).forEach(key => GM_deleteValue(key)); Object.assign(settings, defaultSettings); ['userAgents','userAgentIndex','contentTypes','dataFields','exportFormats','advancedLogging','enableCaptchas'].forEach(key => GM_setValue(key, settings[key])); GM_setValue('customKeywords', {}); GM_setValue('logs', []); GM_setValue('data', []); GM_setValue('visitedUrls', []); GM_setValue('previousCrawls', []); ('#custom-keywords-input').val(''); updateUserAgentSelect(); updateDataUI(); updateLogsUI(); ('#previous-crawls-body').empty(); if (settings.dataVisualization) renderChart(); log('기본 설정으로 초기화되었습니다.', 'success'); } } async function startCrawling() { if (isCrawling) { log('이미 크롤링이 진행 중입니다.', 'warning'); return; } if (!Object.values(settings.searchEngines).some(v => v)) { log('사용할 검색 엔진이 하나도 선택되지 않았습니다.', 'error'); return; } isCrawling = true; GM_setValue('isCrawling', isCrawling); log('크롤링을 시작합니다.', 'info'); setButtonStates(); currentPage = GM_getValue('currentPage', 1); totalPages = settings.maxPages; GM_setValue('totalPages', totalPages); currentDepth = GM_getValue('currentDepth', 0); if (!previousCrawls.some(crawl => crawl.status === '진행 중')) { previousCrawls.push({ date: new Date().toLocaleString(), count: 0, status: '진행 중' }); GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } await crawlPage(currentDepth); } function stopCrawling() { if (!isCrawling) { log('크롤링이 진행 중이지 않습니다.', 'warning'); return; } isCrawling = false; GM_setValue('isCrawling', isCrawling); log('크롤링이 중지되었습니다.', 'warning'); finalizeCrawl('중지됨'); setButtonStates(); } function setButtonStates() { ('#start-crawling').prop('disabled', isCrawling); ('#stop-crawling').prop('disabled', !isCrawling); } async function crawlPage(depth) { try { while (isCrawling && currentPage <= totalPages && depth <= settings.maxDepth) { log(페이지 ${currentPage} 크롤링 시작. (깊이: ${depth}), 'info'); if (settings.respectRobots && !(await checkRobotsTxt(window.location.href))) { log(robots.txt에 의해 크롤링이 금지된 페이지: ${window.location.href}, 'warning'); break; } const results = extractSearchResults(); if (!results) { log('검색 결과를 찾을 수 없습니다.', 'error'); break; } results.forEach(result => { if (!isCrawling) return; const { name, url, description } = result; if (visitedUrls.includes(url) || !isValidURL(url) || (settings.whitelist.length && !settings.whitelist.some(d => url.includes(d))) || settings.blacklist.some(d => url.includes(d)) || (settings.includeKeywords.length && !settings.includeKeywords.some(k => url.includes(k) || description.includes(k))) || (settings.excludeKeywords.some(k => url.includes(k) || description.includes(k))) || (settings.regexInclude.length && !settings.regexInclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description))) || (settings.regexExclude.length && settings.regexExclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description)))) { return; } requestQueue.push({ url, depth }); if (activeRequests >= settings.maxConcurrentRequests) return; }); if (previousCrawls.length && previousCrawls[previousCrawls.length -1].status === '진행 중') { previousCrawls[previousCrawls.length -1].count += results.length; GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } await processQueue(); progress = (currentPage / totalPages) * 100; GM_setValue('progress', progress); updateLogsUI(); if (currentPage < totalPages && isCrawling) { const nextUrl = getNextPageUrl(); if (nextUrl) { currentPage++; GM_setValue('currentPage', currentPage); depth++; GM_setValue('currentDepth', depth); log(다음 페이지로 이동: ${currentPage}, 'info'); await dynamicDelay(); window.location.href = nextUrl; return; } else { log('다음 페이지 링크를 찾을 수 없습니다.', 'error'); break; } } else { break; } } if (isCrawling && currentPage > totalPages && activeRequests === 0 && requestQueue.length === 0) { log('크롤링이 완료되었습니다.', 'success'); finalizeCrawl('완료됨'); isCrawling = false; GM_setValue('isCrawling', isCrawling); setButtonStates(); GM_setValue('progress', 100); ('.status-text').text(진행률: 100% (${currentPage}/${totalPages} 페이지)); } } catch (error) { log(크롤링 중 오류 발생: ${error.message}, 'error'); finalizeCrawl('오류 발생'); } } async function processQueue() { while (requestQueue.length && activeRequests < settings.maxConcurrentRequests && isCrawling) { const { url, depth } = requestQueue.shift(); activeRequests++; handleRequest(url, depth).finally(() => { activeRequests--; processQueue(); }); } } async function handleRequest(url, depth) { if (!isCrawling || visitedUrls.includes(url)) return; try { const response = await fetchWithRetry(url, settings.retryAttempts, settings.retryDelay); const contentType = getContentType(response); if (!isContentTypeAllowed(contentType)) { log(허용되지 않은 콘텐츠 유형(${contentType})의 URL: ${url}, 'warning'); return; } const parser = new DOMParser(); const doc = parser.parseFromString(response.responseText, 'text/html'); const text = doc.body ? doc.body.innerText : ''; if (settings.enableCaptchas && detectCaptcha(doc)) { await handleCaptcha(url); return; } const wc = text.split(/\s+/).length; if (wc < settings.minWordCount || wc > settings.maxWordCount) { log(단어 수 조건에 맞지 않는 URL: ${url} (${wc} 단어), 'warning'); return; } const quality = evaluateContentQuality(text); const lang = doc.documentElement.lang || 'unknown'; const category = classifyKeywords(text); const meta = extractMetaTags(doc); const headings = extractHeadings(doc); if (category !== '기타' || quality > 100) { const entry = { url, quality, language: lang, wordCount: wc, category }; if (settings.dataFields.name) entry.name = extractName(doc) || 'No Title'; if (settings.dataFields.description) entry.description = extractDescription(doc) || 'No Description'; if (settings.dataFields.html) entry.html = response.responseText; if (settings.dataFields.metaTags) entry.metaTags = meta; if (settings.dataFields.headings) entry.headings = headings; data.push(entry); GM_setValue('data', data); updateDataUI(); log(데이터 수집 완료: ${url}, 'success'); if (settings.dataVisualization) renderChart(); } if (depth < settings.maxDepth) { const links = extractLinks(doc); links.forEach(link => { if (!visitedUrls.includes(link) && isValidURL(link)) requestQueue.push({ url: link, depth: depth + 1 }); }); } } catch (error) { log(데이터 수집 실패: ${url} - ${error.message}, 'error'); } visitedUrls.push(url); GM_setValue('visitedUrls', visitedUrls); } async function fetchWithRetry(url, attempts, delayMs) { for (let i = 1; i <= attempts; i++) { try { return await GM_xmlhttpRequestPromise(url); } catch (e) { log(요청 실패 (${i}/${attempts}): ${url} - ${e.message}, 'warning'); if (i < attempts) await delay(delayMs); } } throw new Error('All retry attempts failed'); } function isAnySearchEngineEnabled() { return Object.values(settings.searchEngines).some(v => v); } function classifyKeywords(text) { const kws = combineKeywords(); for (const [cat, keywords] of Object.entries(kws)) { if (keywords.some(kw => text.includes(kw))) return cat; } return '기타'; } function extractSearchResults() { let results = []; const host = window.location.hostname; if (host.includes('google.com') && settings.searchEngines.google) { document.querySelectorAll('div.g, div[data-hveid]').forEach(item => { const name = item.querySelector('h3')?.innerText.trim() || 'No Title'; const url = item.querySelector('a')?.href || ''; const desc = item.querySelector('.aCOpRe, .VwiC3b')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); } else if (host.includes('bing.com') && settings.searchEngines.bing) { document.querySelectorAll('.b_algo').forEach(item => { const name = item.querySelector('h2')?.innerText.trim() || 'No Title'; const url = item.querySelector('h2 a')?.href || ''; const desc = item.querySelector('.b_caption p')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); } // Add similar blocks for other search engines... log(검색 엔진(${host.includes('google.com') ? 'Google' : host.includes('bing.com') ? 'Bing' : '기타'})에서 검색 결과 ${results.length}개 추출됨., 'info'); return results.length ? results : null; } function getNextPageUrl() { const host = window.location.hostname; let next = null; if (host.includes('google.com')) next = document.querySelector('a#pnnext')?.href; else if (host.includes('bing.com')) next = document.querySelector('a.sb_pagN')?.href; // Add similar blocks for other search engines... log(다음 페이지 URL: ${next || '없음'}, 'info'); return next; } function GM_xmlhttpRequestPromise(url) { return new Promise((resolve, reject) => { let reqUrl = settings.proxy ?${settings.proxy}?url=${encodeURIComponent(url)}: url; GM_xmlhttpRequest({ method: "GET", url: reqUrl, headers: { 'User-Agent': getCurrentUserAgent(), 'Accept': '/' }, timeout: settings.timeout, onload: res => (res.status >= 200 && res.status < 300) ? resolve(res) : reject(new Error(HTTP ${res.status})), onerror: () => reject(new Error('Request failed')), ontimeout: () => reject(new Error('Request timed out')) }); }); } function getCurrentUserAgent() { return settings.userAgents[settings.userAgentIndex % settings.userAgents.length]; } function getContentType(res) { return res.responseHeaders.split('\n').find(h => h.toLowerCase().startsWith('content-type'))?.split(':')[1].trim().split(';')[0] || ''; } function isContentTypeAllowed(ct) { if (settings.contentTypes.include.length && !settings.contentTypes.include.includes(ct)) return false; if (settings.contentTypes.exclude.includes(ct)) return false; return true; } function evaluateContentQuality(text) { return text.split(/\s+/).length; } function extractMetaTags(doc) { const metas = {}; doc.querySelectorAll('meta[name], meta[property]').forEach(meta => { const name = meta.getAttribute('name') || meta.getAttribute('property'); const content = meta.getAttribute('content'); if (name && content) metas[name.toLowerCase()] = content; }); return metas; } function extractHeadings(doc) { const hs = []; for (let i = 1; i <= 6; i++) doc.querySelectorAll(h${i}).forEach(h => hs.push(h.innerText.trim())); return hs; } function isValidURL(url) { try { new URL(url); return true; } catch { return false; } } async function checkRobotsTxt(url) { try { const parsed = new URL(url); const robots =${parsed.origin}/robots.txt; const res = await GM_xmlhttpRequestPromise(robots); return new RobotsParser(robots, res.responseText).isAllowed(url, getCurrentUserAgent()); } catch { log('robots.txt 확인 실패, 기본 허용.', 'warning'); return true; } } class RobotsParser { constructor(url, text) { this.rules = []; this.parse(text); } parse(text) { let agents = []; text.split('\n').forEach(line => { const [key, ...val] = line.split(':').map(s => s.trim()); if (!key) return; if (key.toLowerCase() === 'user-agent') { agents = [val.join(':').toLowerCase()]; } else if (['disallow','allow'].includes(key.toLowerCase())) this.rules.push({ agents, type: key.toLowerCase(), path: val.join(':') }); }); } isAllowed(url, ua) { const path = new URL(url).pathname; let allowed = true; this.rules.forEach(rule => { if (rule.agents.includes('*') || rule.agents.includes(ua.toLowerCase())) { if (rule.type === 'disallow' && path.startsWith(rule.path)) allowed = false; if (rule.type === 'allow' && path.startsWith(rule.path)) allowed = true; } }); return allowed; } } function delay(ms) { return new Promise(res => setTimeout(res, ms)); } function formatCustomKeywords(ck) { return Object.entries(ck).map(([c, kws]) =>${c}:${kws.join(',')}).join('\n'); } function getFieldName(field) { return { name: '사이트 이름', url: '주소', description: '설명', html: '📄 HTML 내용', category: '분류', quality: '품질점수', language: '언어', wordCount: '단어 수', metaTags: '메타 태그', headings: '헤딩' }[field]; } function capitalize(str) { return str.charAt(0).toUpperCase() + str.slice(1); } async function handleCaptcha(url) { log(CAPTCHA 감지됨: ${url}, 'warning'); // CAPTCHA 처리 로직 추가 (제한적) // 예: CAPTCHA 해결을 위한 별도의 서비스 연동 또는 수동 해결 유도 } function extractName(doc) { return doc.querySelector('title')?.innerText || 'No Title'; } function extractDescription(doc) { return doc.querySelector('meta[name="description"]')?.getAttribute('content') || 'No Description'; } function extractLinks(doc) { return Array.from(doc.querySelectorAll('a[href]')).map(a => a.href).filter(isValidURL); } function escapeXML(str) { return String(str).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'"); } function init() { createUI(); if (isCrawling) { log('크롤링 재개됨.', 'info'); setButtonStates(); crawlPage(currentDepth); } else log('크롤러 초기화됨.', 'info'); } window.addEventListener('load', init); function makeDraggable() { const container = document.getElementById('crawler-container'); const header = document.getElementById('crawler-header'); let isDragging = false; let startX, startY, initialX, initialY; header.onmousedown = function(e) { isDragging = true; startX = e.clientX; startY = e.clientY; initialX = container.offsetLeft; initialY = container.offsetTop; document.onmousemove = drag; document.onmouseup = stopDrag; return false; }; function drag(e) { if (isDragging) { let dx = e.clientX - startX; let dy = e.clientY - startY; container.style.left = (initialX + dx) + 'px'; container.style.top = (initialY + dy) + 'px'; } } function stopDrag() { isDragging = false; document.onmousemove = null; document.onmouseup = null; } } function evaluateContentQuality(text) { return text.split(/\s+/).length; } async function scheduleCrawling() { const now = new Date(); const [startHour, startMinute] = settings.schedule.startTime.split(':').map(Number); const startTime = new Date(now.getFullYear(), now.getMonth(), now.getDate(), startHour, startMinute, 0); if (startTime < now) startTime.setDate(startTime.getDate() + 1); const initialDelay = startTime - now; setTimeout(() => { startCrawling(); setInterval(startCrawling, settings.schedule.interval); }, initialDelay); log(크롤링 스케줄이 설정되었습니다. 시작 시간: ${startTime.toLocaleString()}, 간격: ${settings.schedule.interval / 60000}분, 'info'); } function finalizeCrawl(status) { if (previousCrawls.length) { previousCrawls[previousCrawls.length -1].status = status; GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } } function updatePreviousCrawlsUI() { ('#previous-crawls-body').html(previousCrawls.map(crawl => ` <tr> <td>{crawl.date} {crawl.status}).join('')); } function updatePreviousCrawlsUI() { $('#previous-crawls-body').html(previousCrawls.map(crawl =>{crawl.count} ${crawl.status} `).join('')); } // 추가적인 기능이나 수정이 필요한 부분은 이곳에 구현하십시오. })(); - Assistant
// ==UserScript== // @name Advanced Web Crawler Enhanced Pro Optimized // @namespace http://tampermonkey.net/ // @version 6.1.0 // @description Optimized version of Advanced Web Crawler Enhanced Pro with enhanced UI controls and additional functionalities // @author Your Name // @match https://www.google.com/search* // @match https://www.bing.com/search* // @match https://search.yahoo.com/search* // @match https://duckduckgo.com/* // @match https://www.baidu.com/s* // @match https://www.yandex.com/search/* // @match https://search.naver.com/search* // @match https://search.daum.net/search* // @match https://www.ask.com/* // @match https://www.ecosia.org/search* // @match https://www.startpage.com/* // @match https://searx.me/* // @match https://www.dogpile.com/* // @match https://www.bing.com/images* // @grant GM_xmlhttpRequest // @grant GM_setValue // @grant GM_getValue // @grant GM_deleteValue // @connect * // @require https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js // @require https://cdn.jsdelivr.net/npm/chart.js // @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js // ==/UserScript== (function() { 'use strict'; const defaultSettings = { /* ... existing settings ... */ }; const settings = Object.assign({}, defaultSettings, { /* ... existing settings loading ... */ }); let visitedUrls = new Set(GM_getValue('visitedUrls', [])); let logs = GM_getValue('logs', []); let data = GM_getValue('data', []); let isCrawling = GM_getValue('isCrawling', false); let currentPage = GM_getValue('currentPage', 1); let totalPages = GM_getValue('totalPages', settings.maxPages); let progress = GM_getValue('progress', 0); let currentDepth = GM_getValue('currentDepth', 0); let activeRequests = 0; let requestQueue = []; let previousCrawls = GM_getValue('previousCrawls', []); const searchEngineHandlers = { google: { extractResults: () => { const results = []; document.querySelectorAll('div.g, div[data-hveid]').forEach(item => { const name = item.querySelector('h3')?.innerText.trim() || 'No Title'; const url = item.querySelector('a')?.href || ''; const desc = item.querySelector('.aCOpRe, .VwiC3b')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: () => document.querySelector('a#pnnext')?.href }, bing: { extractResults: () => { const results = []; document.querySelectorAll('.b_algo').forEach(item => { const name = item.querySelector('h2')?.innerText.trim() || 'No Title'; const url = item.querySelector('h2 a')?.href || ''; const desc = item.querySelector('.b_caption p')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: () => document.querySelector('a.sb_pagN')?.href }, yahoo: { extractResults: () => { const results = []; document.querySelectorAll('.dd.algo').forEach(item => { const name = item.querySelector('h3')?.innerText.trim() || 'No Title'; const url = item.querySelector('a')?.href || ''; const desc = item.querySelector('.compText.aAbs')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: () => document.querySelector('a.next')?.href }, // Add handlers for other search engines... }; function createUI() { /* ... existing UI creation ... */ } function currentTab() { return GM_getValue('currentTab', 'logs'); } function saveCurrentTab(tab) { GM_setValue('currentTab', tab); } function updateUserAgentSelect() { /* ... existing function ... */ } function getCustomKeywordsText() { /* ... existing function ... */ } function parseCustomKeywords(text) { /* ... existing function ... */ } function combineKeywords() { /* ... existing function ... */ } function log(message, type = 'info') { const timestamp = new Date().toLocaleString(); logs.push({ timestamp, message, type }); GM_setValue('logs', logs); updateLogsUI(); if (settings.advancedLogging) { const styles = { 'info': 'color: blue;', 'warning': 'color: orange;', 'error': 'color: red;', 'success': 'color: green; font-weight: bold;' }; console.log(`%c[${timestamp}] [${type.toUpperCase()}] ${message}`, styles[type] || ''); } } function updateLogsUI() { $('#log-messages').html(logs.map(l => `<div>[${l.timestamp}] [${l.type.toUpperCase()}] ${l.message}</div>`).join('')); $('.progress-bar-fill').css('width', `${progress.toFixed(2)}%`); $('.status-text').text(`Progress: ${progress.toFixed(2)}% (${currentPage}/${totalPages} pages)`); } function updateDataUI() { /* ... existing function ... */ } function renderChart() { /* ... existing function ... */ } function escapeHTML(str) { /* ... existing function ... */ } function downloadCSV() { /* ... existing function ... */ } function downloadJSON() { /* ... existing function ... */ } function downloadXML() { /* ... existing function ... */ } function downloadXLS() { /* ... existing function ... */ } function escapeCSV(str) { /* ... existing function ... */ } function filterTable(keyword) { /* ... existing function ... */ } function saveSettings() { /* ... existing function ... */ } function resetSettings() { /* ... existing function ... */ } async function startCrawling() { if (isCrawling) { log('Crawling is already in progress.', 'warning'); return; } if (!isAnySearchEngineEnabled()) { log('No search engines selected.', 'error'); return; } isCrawling = true; GM_setValue('isCrawling', isCrawling); log('Crawling started.', 'info'); setButtonStates(); currentPage = 1; totalPages = settings.maxPages; GM_setValue('totalPages', totalPages); currentDepth = 0; if (!previousCrawls.some(crawl => crawl.status === 'In Progress')) { previousCrawls.push({ date: new Date().toLocaleString(), count: 0, status: 'In Progress' }); GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } await crawlPage(currentDepth); } function stopCrawling() { if (!isCrawling) { log('Crawling is not in progress.', 'warning'); return; } isCrawling = false; GM_setValue('isCrawling', isCrawling); log('Crawling stopped.', 'warning'); finalizeCrawl('Stopped'); setButtonStates(); } function setButtonStates() { $('#start-crawling').prop('disabled', isCrawling); $('#stop-crawling').prop('disabled', !isCrawling); } async function crawlPage(depth) { try { while (isCrawling && currentPage <= totalPages && depth <= settings.maxDepth) { log(`Crawling page ${currentPage} (Depth: ${depth})`, 'info'); if (settings.respectRobots && !(await checkRobotsTxt(window.location.href))) { log(`Blocked by robots.txt: ${window.location.href}`, 'warning'); break; } const host = window.location.hostname; const handlerKey = Object.keys(searchEngineHandlers).find(key => host.includes(key)); const handler = searchEngineHandlers[handlerKey]; if (!handler) { log('No handler for this search engine.', 'error'); break; } const results = handler.extractResults(); if (!results) { log('No search results found.', 'error'); break; } results.forEach(result => { if (!isCrawling) return; const { name, url, description } = result; if (visitedUrls.has(url) || !isValidURL(url) || (settings.whitelist.length && !settings.whitelist.some(d => url.includes(d))) || settings.blacklist.some(d => url.includes(d)) || (settings.includeKeywords.length && !settings.includeKeywords.some(k => url.includes(k) || description.includes(k))) || (settings.excludeKeywords.some(k => url.includes(k) || description.includes(k))) || (settings.regexInclude.length && !settings.regexInclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description))) || (settings.regexExclude.length && settings.regexExclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description)))) { return; } requestQueue.push({ url, depth }); if (activeRequests >= settings.maxConcurrentRequests) return; }); if (previousCrawls.length && previousCrawls[previousCrawls.length -1].status === 'In Progress') { previousCrawls[previousCrawls.length -1].count += results.length; GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } await processQueue(); progress = (currentPage / totalPages) * 100; GM_setValue('progress', progress); updateLogsUI(); if (currentPage < totalPages && isCrawling) { const nextUrl = handler.getNextPageUrl(); if (nextUrl) { currentPage++; GM_setValue('currentPage', currentPage); depth++; GM_setValue('currentDepth', depth); log(`Navigating to next page: ${currentPage}`, 'info'); await dynamicDelay(); const response = await fetchPage(nextUrl); const parser = new DOMParser(); const doc = parser.parseFromString(response, 'text/html'); Object.assign(document, doc); continue; } else { log('Next page link not found.', 'error'); break; } } else { break; } } if (isCrawling && currentPage > totalPages && activeRequests === 0 && requestQueue.length === 0) { log('Crawling completed.', 'success'); finalizeCrawl('Completed'); isCrawling = false; GM_setValue('isCrawling', isCrawling); setButtonStates(); GM_setValue('progress', 100); $('.progress-bar-fill').css('width', `100%`); $('.status-text').text(`Progress: 100% (${currentPage}/${totalPages} pages)`); } } catch (error) { log(`Error during crawling: ${error.message}`, 'error'); finalizeCrawl('Error'); } } async function processQueue() { while (requestQueue.length && activeRequests < settings.maxConcurrentRequests && isCrawling) { const { url, depth } = requestQueue.shift(); activeRequests++; handleRequest(url, depth).finally(() => { activeRequests--; processQueue(); }); } } async function handleRequest(url, depth) { if (!isCrawling || visitedUrls.has(url)) return; try { const response = await fetchWithRetry(url, settings.retryAttempts, settings.retryDelay); const contentType = getContentType(response); if (!isContentTypeAllowed(contentType)) { log(`Disallowed content type (${contentType}): ${url}`, 'warning'); return; } const parser = new DOMParser(); const doc = parser.parseFromString(response.responseText, 'text/html'); const text = doc.body ? doc.body.innerText : ''; if (settings.enableCaptchas && detectCaptcha(doc)) { await handleCaptcha(url); return; } const wc = text.split(/\s+/).length; if (wc < settings.minWordCount || wc > settings.maxWordCount) { log(`Word count out of range for URL: ${url} (${wc} words)`, 'warning'); return; } const quality = evaluateContentQuality(text); const lang = doc.documentElement.lang || 'unknown'; const category = classifyKeywords(text); const meta = extractMetaTags(doc); const headings = extractHeadings(doc); if (category !== 'Other' || quality > 100) { const entry = { url, quality, language: lang, wordCount: wc, category }; if (settings.dataFields.name) entry.name = extractName(doc) || 'No Title'; if (settings.dataFields.description) entry.description = extractDescription(doc) || 'No Description'; if (settings.dataFields.html) entry.html = response.responseText; if (settings.dataFields.metaTags) entry.metaTags = meta; if (settings.dataFields.headings) entry.headings = headings; data.push(entry); GM_setValue('data', Array.from(data)); updateDataUI(); log(`Data collected: ${url}`, 'success'); if (settings.dataVisualization) renderChart(); } if (depth < settings.maxDepth) { const links = extractLinks(doc); links.forEach(link => { if (!visitedUrls.has(link) && isValidURL(link)) requestQueue.push({ url: link, depth: depth + 1 }); }); } } catch (error) { log(`Failed to collect data: ${url} - ${error.message}`, 'error'); } visitedUrls.add(url); GM_setValue('visitedUrls', Array.from(visitedUrls)); } async function fetchWithRetry(url, attempts, delayMs) { for (let i = 1; i <= attempts; i++) { try { const res = await GM_xmlhttpRequestPromise(url); return res; } catch (e) { log(`Request failed (${i}/${attempts}): ${url} - ${e.message}`, 'warning'); if (i < attempts) await delay(delayMs); } } throw new Error('All retry attempts failed'); } function isAnySearchEngineEnabled() { return Object.values(settings.searchEngines).some(v => v); } function classifyKeywords(text) { const kws = combineKeywords(); for (const [cat, keywords] of Object.entries(kws)) { if (keywords.some(kw => text.includes(kw))) return cat; } return 'Other'; } async function extractSearchResults() { const host = window.location.hostname; const handler = Object.keys(searchEngineHandlers).find(key => host.includes(key)); if (handler && settings.searchEngines[handler]) { return searchEngineHandlers[handler].extractResults(); } log('No handler available for the current search engine.', 'error'); return null; } function getNextPageUrl() { const host = window.location.hostname; const handler = Object.keys(searchEngineHandlers).find(key => host.includes(key)); if (handler && settings.searchEngines[handler]) { return searchEngineHandlers[handler].getNextPageUrl(); } log('No handler for getting next page URL.', 'error'); return null; } function GM_xmlhttpRequestPromise(url) { return new Promise((resolve, reject) => { let reqUrl = settings.proxy ? `${settings.proxy}?url=${encodeURIComponent(url)}` : url; GM_xmlhttpRequest({ method: "GET", url: reqUrl, headers: { 'User-Agent': getCurrentUserAgent(), 'Accept': '*/*' }, timeout: settings.timeout, onload: res => (res.status >= 200 && res.status < 300) ? resolve(res) : reject(new Error(`HTTP ${res.status}`)), onerror: () => reject(new Error('Request failed')), ontimeout: () => reject(new Error('Request timed out')) }); }); } function getCurrentUserAgent() { return settings.userAgents[settings.userAgentIndex % settings.userAgents.length]; } function getContentType(res) { return res.responseHeaders.split('\n').find(h => h.toLowerCase().startsWith('content-type'))?.split(':')[1].trim().split(';')[0] || ''; } function isContentTypeAllowed(ct) { if (settings.contentTypes.include.length && !settings.contentTypes.include.includes(ct)) return false; if (settings.contentTypes.exclude.includes(ct)) return false; return true; } function evaluateContentQuality(text) { return text.split(/\s+/).length; } function extractMetaTags(doc) { const metas = {}; doc.querySelectorAll('meta[name], meta[property]').forEach(meta => { const name = meta.getAttribute('name') || meta.getAttribute('property'); const content = meta.getAttribute('content'); if (name && content) metas[name.toLowerCase()] = content; }); return metas; } function extractHeadings(doc) { const hs = []; for (let i = 1; i <= 6; i++) doc.querySelectorAll(`h${i}`).forEach(h => hs.push(h.innerText.trim())); return hs; } function isValidURL(url) { try { new URL(url); return true; } catch { return false; } } async function checkRobotsTxt(url) { try { const parsed = new URL(url); const robots = `${parsed.origin}/robots.txt`; const res = await GM_xmlhttpRequestPromise(robots); return new RobotsParser(robots, res.responseText).isAllowed(url, getCurrentUserAgent()); } catch { log('Failed to check robots.txt, allowing by default.', 'warning'); return true; } } class RobotsParser { constructor(url, text) { this.rules = []; this.parse(text); } parse(text) { let agents = []; text.split('\n').forEach(line => { const [key, ...val] = line.split(':').map(s => s.trim()); if (!key) return; if (key.toLowerCase() === 'user-agent') { agents = [val.join(':').toLowerCase()]; } else if (['disallow','allow'].includes(key.toLowerCase())) this.rules.push({ agents, type: key.toLowerCase(), path: val.join(':') }); }); } isAllowed(url, ua) { const path = new URL(url).pathname; let allowed = true; this.rules.forEach(rule => { if (rule.agents.includes('*') || rule.agents.includes(ua.toLowerCase())) { if (rule.type === 'disallow' && path.startsWith(rule.path)) allowed = false; if (rule.type === 'allow' && path.startsWith(rule.path)) allowed = true; } }); return allowed; } } function delay(ms) { return new Promise(res => setTimeout(res, ms)); } function formatCustomKeywords(ck) { return Object.entries(ck).map(([c, kws]) => `${c}:${kws.join(',')}`).join('\n'); } function getFieldName(field) { return { name: 'Site Name', url: 'URL', description: 'Description', html: 'HTML Content', category: 'Category', quality: 'Quality Score', language: 'Language', wordCount: 'Word Count', metaTags: 'Meta Tags', headings: 'Headings' }[field]; } function capitalize(str) { return str.charAt(0).toUpperCase() + str.slice(1); } async function handleCaptcha(url) { /* CAPTCHA handling logic */ } function extractName(doc) { return doc.querySelector('title')?.innerText || 'No Title'; } function extractDescription(doc) { return doc.querySelector('meta[name="description"]')?.getAttribute('content') || 'No Description'; } function extractLinks(doc) { return Array.from(doc.querySelectorAll('a[href]')).map(a => a.href).filter(isValidURL); } function escapeXML(str) { return String(str).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'"); } function init() { createUI(); if (isCrawling) { log('Resuming crawling.', 'info'); setButtonStates(); crawlPage(currentDepth); } else log('Crawler initialized.', 'info'); } window.addEventListener('load', init); function makeDraggable() { /* ... existing function ... */ } function finalizeCrawl(status) { if (previousCrawls.length) { previousCrawls[previousCrawls.length -1].status = status; GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } } function updatePreviousCrawlsUI() { $('#previous-crawls-body').html(previousCrawls.map(crawl => ` <tr> <td>${crawl.date}</td> <td>${crawl.count}</td> <td>${crawl.status}</td> </tr> `).join('')); } async function dynamicDelay() { const base = settings.delay; const random = Math.floor(Math.random() * settings.delayRandom); await delay(base + random); } })();