// ==UserScript== // @name Advanced Web Crawler Enhanced Pro Optimized // @namespace…
- User
// ==UserScript== // @name Advanced Web Crawler Enhanced Pro Optimized // @namespace http://tampermonkey.net/ // @version 6.0.3 // @description Optimized version of Advanced Web Crawler Enhanced Pro with enhanced UI controls and additional functionalities // @author Your Name // @match https://www.google.com/search* // @match https://www.bing.com/search* // @match https://search.yahoo.com/search* // @match https://duckduckgo.com/* // @match https://www.baidu.com/s* // @match https://www.yandex.com/search/* // @match https://search.naver.com/search* // @match https://search.daum.net/search* // @match https://www.ask.com/* // @match https://www.ecosia.org/search* // @match https://www.startpage.com/* // @match https://searx.me/* // @match https://www.dogpile.com/* // @match https://www.bing.com/images* // @grant GM_xmlhttpRequest // @grant GM_setValue // @grant GM_getValue // @grant GM_deleteValue // @connect * // @require https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js // @require https://cdn.jsdelivr.net/npm/chart.js // @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js // ==/UserScript==
(function() { 'use strict'; const defaultSettings = { delay: 2000, delayRandom: 500, maxPages: 3, userAgents: [ 'AdvancedWebCrawler/6.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)', 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15', 'Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15', ], userAgentIndex: 0, blacklist: [], whitelist: [], contentTypes: { include: ['text/html'], exclude: ['application/pdf', 'image/jpeg', 'image/png', 'video/mp4'] }, timeout: 8000, includeKeywords: [], excludeKeywords: [], language: [], minWordCount: 100, maxWordCount: 10000, maxDepth: 2, proxy: '', regexInclude: [], regexExclude: [], dataFields: { name: true, url: true, description: true, html: true, category: true, quality: true, language: true, wordCount: true, metaTags: false, headings: false }, schedule: { enabled: false, startTime: '00:00', interval: 3600000 }, searchEngines: { google: true, bing: true, yahoo: true, duckduckgo: true, baidu: true, yandex: true, naver: true, daum: true, ask: true, ecosia: true, startpage: true, searx: true, dogpile: true, bingimages: true }, retryAttempts: 3, retryDelay: 2000, respectRobots: true, maxConcurrentRequests: 3, dataVisualization: true, exportFormats: { csv: true, json: true, xml: true, xls: true }, advancedLogging: true, enableCaptchas: false }; const settings = Object.assign({}, defaultSettings, { delay: parseInt(GM_getValue('delay', defaultSettings.delay), 10), delayRandom: parseInt(GM_getValue('delayRandom', defaultSettings.delayRandom), 10), maxPages: parseInt(GM_getValue('maxPages', defaultSettings.maxPages), 10), userAgents: GM_getValue('userAgents', defaultSettings.userAgents), userAgentIndex: parseInt(GM_getValue('userAgentIndex', defaultSettings.userAgentIndex), 10), blacklist: GM_getValue('blacklist', defaultSettings.blacklist), whitelist: GM_getValue('whitelist', defaultSettings.whitelist), contentTypes: GM_getValue('contentTypes', defaultSettings.contentTypes), timeout: parseInt(GM_getValue('timeout', defaultSettings.timeout), 10), includeKeywords: GM_getValue('includeKeywords', defaultSettings.includeKeywords), excludeKeywords: GM_getValue('excludeKeywords', defaultSettings.excludeKeywords), regexInclude: GM_getValue('regexInclude', defaultSettings.regexInclude), regexExclude: GM_getValue('regexExclude', defaultSettings.regexExclude), language: GM_getValue('language', defaultSettings.language), minWordCount: parseInt(GM_getValue('minWordCount', defaultSettings.minWordCount), 10), maxWordCount: parseInt(GM_getValue('maxWordCount', defaultSettings.maxWordCount), 10), maxDepth: parseInt(GM_getValue('maxDepth', defaultSettings.maxDepth), 10), proxy: GM_getValue('proxy', defaultSettings.proxy), dataFields: GM_getValue('dataFields', defaultSettings.dataFields), schedule: GM_getValue('schedule', defaultSettings.schedule), searchEngines: GM_getValue('searchEngines', defaultSettings.searchEngines), retryAttempts: parseInt(GM_getValue('retryAttempts', defaultSettings.retryAttempts), 10), retryDelay: parseInt(GM_getValue('retryDelay', defaultSettings.retryDelay), 10), respectRobots: GM_getValue('respectRobots', defaultSettings.respectRobots), maxConcurrentRequests: parseInt(GM_getValue('maxConcurrentRequests', defaultSettings.maxConcurrentRequests), 10), dataVisualization: GM_getValue('dataVisualization', defaultSettings.dataVisualization), exportFormats: GM_getValue('exportFormats', defaultSettings.exportFormats), advancedLogging: GM_getValue('advancedLogging', defaultSettings.advancedLogging), enableCaptchas: GM_getValue('enableCaptchas', defaultSettings.enableCaptchas), customKeywords: GM_getValue('customKeywords', {}) }); let visitedUrls = GM_getValue('visitedUrls', []); let logs = GM_getValue('logs', []); let data = GM_getValue('data', []); let isCrawling = GM_getValue('isCrawling', false); let currentPage = GM_getValue('currentPage', 1); let totalPages = GM_getValue('totalPages', settings.maxPages); let progress = GM_getValue('progress', 0); let currentDepth = GM_getValue('currentDepth', 0); let activeRequests = 0; let requestQueue = []; let previousCrawls = GM_getValue('previousCrawls', []); function createUI() { const style =
#crawler-container { position: fixed; top: 20px; right: 20px; width: 400px; height: 500px; background: #fff; border: 1px solid #ccc; z-index: 10000; font-family: Arial, sans-serif; box-shadow: 0 4px 8px rgba(0,0,0,0.2); overflow: auto; resize: both; border-radius: 8px; min-width: 300px; min-height: 300px; } #crawler-header { display: flex; justify-content: space-between; align-items: center; background: #f1f1f1; border-bottom: 1px solid #ccc; padding: 5px 10px; cursor: move; } #crawler-minimize { background: none; border: none; font-size: 18px; cursor: pointer; } #crawler-tabs { display: flex; background: #f1f1f1; border-bottom: 1px solid #ccc; } .crawler-tab { flex: 1; padding: 10px; text-align: center; cursor: pointer; font-weight: bold; } .crawler-tab.active { background: #ddd; } #crawler-content { padding: 10px; overflow: auto; height: calc(100% - 50px); } table { width: 100%; border-collapse: collapse; } th, td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; } th { background-color: #f2f2f2; cursor: pointer; } tr:hover { background-color: #f5f5f5; } .progress-bar { width: 100%; background-color: #f3f3f3; border: 1px solid #ccc; border-radius: 5px; overflow: hidden; margin-bottom: 10px; } .progress-bar-fill { height: 20px; background-color: #4caf50; width: 0%; transition: width 0.2s; } .modal { display: none; position: fixed; z-index: 10001; left: 0; top: 0; width: 100%; height: 100%; overflow: auto; background-color: rgba(0,0,0,0.4); } .modal-content { background-color: #fff; margin: 5% auto; padding: 20px; border: 1px solid #888; width: 80%; max-height: 80%; overflow-y: auto; border-radius: 8px; } .close { color: #aaa; float: right; font-size: 28px; font-weight: bold; cursor: pointer; } .control-buttons { margin: 10px 0; } .control-buttons button { padding: 8px 12px; margin-right: 8px; cursor: pointer; border: none; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px; } .control-buttons button:disabled { background-color: #a5d6a7; cursor: not-allowed; } .status-text { margin-top: 10px; font-weight: bold; font-size: 14px; } .settings-section { margin-bottom: 15px; } .settings-section label { display: block; margin-bottom: 5px; font-weight: bold; } .settings-section input[type="text"], .settings-section input[type="number"], .settings-section textarea, .settings-section select { width: 100%; padding: 5px; box-sizing: border-box; border: 1px solid #ccc; border-radius: 4px; font-size: 14px; } .settings-section .checkbox-group { display: flex; flex-wrap: wrap; } .settings-section .checkbox-group label { margin-right: 10px; font-size: 14px; } .settings-section .radio-group { display: flex; flex-wrap: wrap; } .settings-section .radio-group label { margin-right: 10px; font-size: 14px; } .tooltip { position: relative; display: inline-block; cursor: help; font-size: 14px; } .tooltip .tooltiptext { visibility: hidden; width: 220px; background-color: #555; color: #fff; text-align: center; border-radius: 6px; padding: 5px; position: absolute; z-index: 1; bottom: 125%; left: 50%; margin-left: -110px; opacity: 0; transition: opacity 0.3s; } .tooltip:hover .tooltiptext { visibility: visible; opacity: 1; } #crawler-container::-webkit-resizer { background: #f1f1f1; border: none; } #data-visualization { margin-top: 20px; } #data-chart { width: 100%; height: 300px; } #modal-html-content { white-space: pre-wrap; word-wrap: break-word; background-color: #f9f9f9; padding: 10px; border-radius: 4px; } #crawler-tabs .crawler-tab:last-child { flex: 1; } #previous-crawls { margin-top: 20px; } #previous-crawls table { width: 100%; } #previous-crawls th, #previous-crawls td { padding: 5px; } #reset-data { background-color: #f44336; }; {style}); const container = $(📜 로그 📊 데이터 ⚙️ 설정 📁 기록 ─ 진행률: {currentPage}/{isCrawling ? 'disabled' : ''}>▶ 시작 <button id="stop-crawling" title="크롤링을 중지합니다." {settings.exportFormats.csv ?<button id="download-csv" title="CSV 다운로드">📥 CSV</button>: ''} {settings.exportFormats.xml ?<button id="download-xml" title="XML 다운로드">📥 XML</button>: ''} {settings.exportFormats.xls ? `<button id="download-xls" title="XLS 다운로드">📥 XLS</button>` : ''} <button id="reset-data" title="모든 크롤링 데이터를 초기화합니다.">🗑️ 초기화</button> </div> <div id="log-messages" style="height: 200px; overflow-y: auto; background: #f9f9f9; padding: 10px; border: 1px solid #ccc; border-radius: 4px;">로그 내용이 여기에 표시됩니다.</div> </div> <div id="tab-data" style="display: {currentTab() === 'data' ? 'block' : 'none'};"> {settings.dataFields.url ? '주소' : ''} {settings.dataFields.html ? '📄 HTML 내용' : ''} {settings.dataFields.quality ? '품질점수' : ''} {settings.dataFields.wordCount ? '단어 수' : ''} {settings.dataFields.headings ? '헤딩' : ''} {currentTab() === 'settings' ? 'block' : 'none'};"> 설정 크롤링 딜레이 (ms): ℹ️ 기본 딜레이과 딜레이 랜덤 값을 조합하여 요청 사이의 딜레이를 설정합니다. 딜레이 랜덤 범위 (ms): ℹ️ 딜레이에 무작위 요소를 추가하여 보다 자연스러운 요청을 합니다. 최대 크롤링 페이지 수: ℹ️ 크롤링할 최대 페이지 수를 설정하세요. 사용자 에이전트(User-Agent): ℹ️ 크롤링에 사용할 사용자 에이전트를 선택하거나 추가할 수 있습니다. {settings.userAgents.map((ua, idx) => `<option value="{idx}" {ua}).join('')} </select> <button id="add-user-agent" style="margin-top: 5px; padding: 5px 10px; background-color: #2196F3; color: white; border-radius: 4px; font-size: 12px;">추가</button> <button id="remove-user-agent" style="margin-top: 5px; padding: 5px 10px; background-color: #f44336; color: white; border-radius: 4px; font-size: 12px;">제거</button> </div> <div class="settings-section"> <label>콘텐츠 유형:</label> <div> <strong>포함할 유형:</strong> <textarea id="content-include-input" rows="3">${settings.contentTypes.include.join(', ')}</textarea> </div> <div> <strong>제외할 유형:</strong> <textarea id="content-exclude-input" rows="3">${settings.contentTypes.exclude.join(', ')}</textarea> </div> </div> <div class="settings-section"> <label for="timeout-input">크롤링 타임아웃 (ms): <span class="tooltip">ℹ️ <span class="tooltiptext">요청이 완료되기까지 기다릴 최대 시간을 밀리초 단위로 설정합니다.</span> </span> </label> <input type="number" id="timeout-input" value="${settings.timeout}" min="1000" step="1000"> </div> <div class="settings-section"> <label for="include-keywords-input">포함할 키워드: <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링할 페이지에 반드시 포함되어야 하는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="include-keywords-input" rows="3">${settings.includeKeywords.join('\n')}</textarea> </div> <div class="settings-section"> <label for="exclude-keywords-input">제외할 키워드: <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링할 페이지에 포함되면 안 되는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="exclude-keywords-input" rows="3">${settings.excludeKeywords.join('\n')}</textarea> </div> <div class="settings-section"> <label for="regex-include-input">포함할 키워드 (정규 표현식): <span class="tooltip">ℹ️ <span class="tooltiptext">포함할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="regex-include-input" rows="3">${settings.regexInclude.join('\n')}</textarea> </div> <div class="settings-section"> <label for="regex-exclude-input">제외할 키워드 (정규 표현식): <span class="tooltip">ℹ️ <span class="tooltiptext">제외할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="regex-exclude-input" rows="3">${settings.regexExclude.join('\n')}</textarea> </div> <div class="settings-section"> <label for="language-select">언어 필터링:</label> <select id="language-select" multiple size="10"> ${[ { value: 'en', text: '영어 (en)' }, { value: 'ko', text: '한국어 (ko)' }, { value: 'zh', text: '중국어 (zh)' }, { value: 'ja', text: '일본어 (ja)' }, { value: 'ru', text: '러시아어 (ru)' }, { value: 'es', text: '스페인어 (es)' }, { value: 'fr', text: '프랑스어 (fr)' }, { value: 'de', text: '독일어 (de)' }, { value: 'it', text: '이탈리아어 (it)' }, { value: 'pt', text: '포르투갈어 (pt)' }, { value: 'ar', text: '아랍어 (ar)' }, { value: 'hi', text: '힌디어 (hi)' }, { value: 'bn', text: '벵골어 (bn)' }, { value: 'pa', text: '펀자브어 (pa)' }, { value: 'nl', text: '네덜란드어 (nl)' }, { value: 'sv', text: '스웨덴어 (sv)' }, { value: 'tr', text: '터키어 (tr)' }, { value: 'vi', text: '베트남어 (vi)' }, { value: 'pl', text: '폴란드어 (pl)' }, { value: 'th', text: '태국어 (th)' } ].map(lang =><option value="{settings.language.includes(lang.value) ? 'selected' : ''}>{lang.text}</option>`).join('')} </select> </div> <div class="settings-section"> <label for="min-word-count-input">최소 단어 수: <span class="tooltip">ℹ️ <span class="tooltiptext">수집할 페이지의 최소 단어 수를 설정하세요.</span> </span> </label> <input type="number" id="min-word-count-input" value="{settings.minWordCount}" min="10" step="10"> 최대 단어 수: ℹ️ 수집할 페이지의 최대 단어 수를 설정하세요. 최대 크롤링 깊이: ℹ️ 링크를 추적할 최대 깊이를 설정하세요. 프록시 서버: ℹ️ 프록시 서버 주소를 입력하세요. 예: http://proxyserver:port 수집할 데이터 필드: {Object.keys(settings.dataFields).map(field => `<label><input type="checkbox" id="field-{field}" {getFieldName(field)}).join('')} </div> </div> <div class="settings-section"> <label for="retry-attempts-input">재시도 횟수: <span class="tooltip">ℹ️ <span class="tooltiptext">요청 실패 시 재시도할 횟수를 설정하세요.</span> </span> </label> <input type="number" id="retry-attempts-input" value="${settings.retryAttempts}" min="0" step="1"> <label for="retry-delay-input">재시도 딜레이 (ms): <span class="tooltip">ℹ️ <span class="tooltiptext">재시도 시 대기할 시간을 밀리초 단위로 설정하세요.</span> </span> </label> <input type="number" id="retry-delay-input" value="${settings.retryDelay}" min="1000" step="500"> </div> <div class="settings-section"> <label>크롤링 스케줄링:</label> <div class="checkbox-group"> <label><input type="checkbox" id="schedule-enabled" ${settings.schedule.enabled ? 'checked' : ''}> 스케줄링 활성화</label> </div> <div id="schedule-options" style="display: ${settings.schedule.enabled ? 'block' : 'none'}; margin-top: 10px;"> <label for="start-time-input">시작 시간 (HH:MM): <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링을 시작할 시간을 설정하세요.</span> </span> </label> <input type="time" id="start-time-input" value="${settings.schedule.startTime}"> <label for="interval-input">크롤링 간격 (분): <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링 작업 간의 간격을 분 단위로 설정하세요.</span> </span> </label> <input type="number" id="interval-input" value="${settings.schedule.interval / 60000}" min="1" step="1"> </div> </div> <div class="settings-section"> <label>크롤링할 검색 엔진 선택:</label> <div class="checkbox-group"> ${Object.keys(settings.searchEngines).map(engine =><input type="checkbox" id="engine-{settings.searchEngines[engine] ? 'checked' : ''}> {capitalize(engine)}</label>`).join('')} </div> </div> <div class="settings-section"> <label for="blacklist-input">블랙리스트 도메인: <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링에서 제외할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="blacklist-input" rows="5">{settings.blacklist.join('\n')} 화이트리스트 도메인: ℹ️ 크롤링을 허용할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다. {settings.respectRobots ? 'checked' : ''}> 동시 요청 최대 수: ℹ️ 한 번에 요청할 수 있는 최대 동시 요청 수를 설정하세요. 데이터 시각화 활성화: ℹ️ 수집된 데이터를 시각화하여 그래프로 표시합니다. <input type="checkbox" id="data-visualization-checkbox" {settings.enableCaptchas ? 'checked' : ''}> 데이터 내보내기 형식: {Object.keys(settings.exportFormats).map(format => `<label><input type="checkbox" id="export-{format}" {format.toUpperCase()}).join('')} </div> </div> <div class="settings-section"> <label for="advanced-logging-checkbox">고급 로깅 활성화: <span class="tooltip">ℹ️ <span class="tooltiptext">추가적인 로깅 정보를 활성화합니다.</span> </span> </label> <input type="checkbox" id="advanced-logging-checkbox" ${settings.advancedLogging ? 'checked' : ''}> </div> <div class="settings-section"> <label for="custom-keywords-input">사용자 지정 키워드 분류: <span class="tooltip">ℹ️ <span class="tooltiptext">사용자 지정 키워드를 추가하여 데이터 분류를 향상시킵니다. 형식: 카테고리:키워드1,키워드2,...</span> </span> </label> <textarea id="custom-keywords-input" rows="5">${formatCustomKeywords(settings.customKeywords)}</textarea> </div> <button id="save-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">저장</button> <button id="reset-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">초기화</button> </div> <div id="tab-previous-crawls" style="display: ${currentTab() === 'previous-crawls' ? 'block' : 'none'};"> <h3>이전 크롤링 기록</h3> <button id="clear-previous-crawls" style="padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px; margin-bottom: 10px;">기록 초기화</button> <table> <thead> <tr> <th>날짜</th> <th>크롤링 수</th> <th>상태</th> </tr> </thead> <tbody id="previous-crawls-body"> ${previousCrawls.map(crawl =>{crawl.count} {crawl.status}</td> </tr> `).join('')} </tbody> </table> </div> </div> <div id="html-modal" class="modal"> <div class="modal-content"> <span class="close">×</span> <pre id="modal-html-content"></pre> </div> </div> </div> `); ('body').append(container); ('#crawler-minimize').on('click', function() { ('#crawler-content').toggle(); const isHidden = ('#crawler-content').is(':hidden'); (this).text(isHidden ? '☐' : '─'); log(크롤러 창이 ${isHidden ? '최소화' : '확대'}되었습니다., 'info'); }); ('.crawler-tab').on('click', function() { const tab = (this).data('tab'); (this).addClass('active'); ('#crawler-content > div').hide(); (#tab-${tab}).show(); saveCurrentTab(tab); }); (document).on('keydown', function(e) { if (e.ctrlKey) { const actions = { '1': 'logs', '2': 'data', '3': 'settings', '4': 'previous-crawls', 's': 'save', 'r': 'reset', 'p': 'start', 'q': 'stop', 'c': 'csv', 'j': 'json', 'x': 'xml', 'l': 'xls' }; const action = actions[e.key]; if (action) { e.preventDefault(); switch(action) { case 'logs': ('.crawler-tab[data-tab="logs"]').click(); break; case 'data': ('.crawler-tab[data-tab="settings"]').click(); break; case 'previous-crawls': ('.crawler-tab[data-tab="previous-crawls"]').click(); break; case 'save': saveSettings(); break; case 'reset': resetSettings(); break; case 'start': if (!isCrawling) startCrawling(); break; case 'stop': if (isCrawling) stopCrawling(); break; case 'csv': settings.exportFormats.csv && downloadCSV(); break; case 'json': settings.exportFormats.json && downloadJSON(); break; case 'xml': settings.exportFormats.xml && downloadXML(); break; case 'xls': settings.exportFormats.xls && downloadXLS(); break; } } } }); ('#save-settings').on('click', saveSettings); ('#reset-settings').on('click', resetSettings); ('#add-user-agent').on('click', function() { const ua = prompt("새 사용자 에이전트를 입력하세요:"); if (ua) { settings.userAgents.push(ua); GM_setValue('userAgents', settings.userAgents); updateUserAgentSelect(); log('새 사용자 에이전트가 추가되었습니다.', 'success'); } }); ('#remove-user-agent').on('click', function() { const selected = ('#user-agent-select').val(); if (selected && selected.length) { settings.userAgents = settings.userAgents.filter((_, idx) => !selected.includes(idx.toString())); GM_setValue('userAgents', settings.userAgents); settings.userAgentIndex = 0; GM_setValue('userAgentIndex', settings.userAgentIndex); updateUserAgentSelect(); log('선택한 사용자 에이전트가 제거되었습니다.', 'success'); } else alert('제거할 사용자 에이전트를 선택하세요.'); }); ('#download-csv').on('click', downloadCSV); ('#download-json').on('click', downloadJSON); ('#download-xml').on('click', downloadXML); ('#download-xls').on('click', downloadXLS); ('#search-filter').on('input', function() { filterTable((this).val()); }); ('#schedule-enabled').on('change', function() { ('#schedule-options').toggle((this).is(':checked')); }); ('#data-visualization-checkbox').on('change', function() { settings.dataVisualization = ('#data-visualization').toggle(settings.dataVisualization); log('데이터 시각화 설정이 변경되었습니다.', 'success'); if (settings.dataVisualization) renderChart(); }); ('#enable-captchas-checkbox').on('change', function() { settings.enableCaptchas = (this).is(':checked'); GM_setValue('enableCaptchas', settings.enableCaptchas); log('CAPTCHA 처리 설정이 변경되었습니다.', 'success'); }); ('.view-html').on('click', function() { const html = decodeURIComponent((this).data('html')); ('#modal-html-content').text(html); ('#html-modal').show(); }); ('.close').on('click', function() { ('#html-modal').hide(); }); (window).on('click', function(e) { if (e.target.id === 'html-modal') ('#html-modal').hide(); }); ('#start-crawling').on('click', startCrawling); ('#stop-crawling').on('click', stopCrawling); ('#download-csv').on('click', downloadCSV); ('#download-json').on('click', downloadJSON); ('#download-xml').on('click', downloadXML); ('#download-xls').on('click', downloadXLS); ('#clear-previous-crawls').on('click', function() { if (confirm("이전 크롤링 기록을 모두 초기화하시겠습니까?")) { previousCrawls = []; GM_setValue('previousCrawls', previousCrawls); ('#previous-crawls-body').empty(); log('이전 크롤링 기록이 초기화되었습니다.', 'success'); } }); ('#reset-data').on('click', function() { if (confirm("모든 크롤링 데이터를 초기화하시겠습니까?")) { data = []; visitedUrls = []; GM_setValue('data', data); GM_setValue('visitedUrls', visitedUrls); updateDataUI(); log('크롤링 데이터가 초기화되었습니다.', 'success'); } }); updateLogsUI(); updateDataUI(); if (settings.dataVisualization) renderChart(); updatePreviousCrawlsUI(); makeDraggable(); } function currentTab() { return GM_getValue('currentTab', 'logs'); } function saveCurrentTab(tab) { GM_setValue('currentTab', tab); } function updateUserAgentSelect() { ('#user-agent-select').empty(); settings.userAgents.forEach((ua, idx) => { ('#user-agent-select').append(`<option value="{idx}" {ua}); }); } function getCustomKeywordsText() { const ck = settings.customKeywords || {}; return Object.entries(ck).map(([cat, kws]) =>{kws.join(',')}).join('\n'); } function parseCustomKeywords(text) { const lines = text.split('\n').map(l => l.trim()).filter(l => l); const ck = {}; lines.forEach(line => { const [cat, kws] = line.split(':'); if (cat && kws) ck[cat.trim()] = kws.split(',').map(kw => kw.trim()).filter(kw => kw); }); return ck; } function combineKeywords() { const standard = { '교육': ['학교', '교육', '학생', '교수', '강의'], '기술': ['기술', '소프트웨어', '하드웨어', '프로그래밍', '개발'], '뉴스': ['뉴스', '속보', '기사', '보도'], '엔터테인먼트': ['영화', '음악', '연예', '게임', '드라마'], '스포츠': ['축구', '야구', '농구', '경기', '토너먼트'], '비즈니스': ['비즈니스', '경제', '주식', '투자', '시장'], '건강': ['건강', '의료', '헬스', '운동', '영양'], '과학': ['과학', '연구', '실험', '발견', '이론'], '문학': ['소설', '시', '문학', '작가', '책'], '여행': ['여행', '관광', '숙박', '비행기', '여권'] }; const custom = settings.customKeywords || {}; for (const [cat, kws] of Object.entries(custom)) { standard[cat] = standard[cat] ? [...new Set([...standard[cat], ...kws])] : kws; } return standard; } function log(message, type = 'info') { const timestamp = new Date().toLocaleString(); logs.push({ timestamp, message, type }); GM_setValue('logs', logs); updateLogsUI(); if (settings.advancedLogging) { const styles = { 'info': 'color: blue;', 'warning': 'color: orange;', 'error': 'color: red;', 'success': 'color: green; font-weight: bold;' }; console.log(%c[{type.toUpperCase()}] {message}`, styles[type] || ''); } } function updateLogsUI() { ('#log-messages').html(logs.map(l =><div>[${l.timestamp}] [${l.type.toUpperCase()}] ${l.message}</div>).join('')); {progress.toFixed(2)}%); $('.status-text').text(진행률: {currentPage}/{totalPages} 페이지)`); } function updateDataUI() { const filter = ('#search-filter').val().toLowerCase(); const filtered = filter ? data.filter(item => Object.keys(settings.dataFields).some(field => settings.dataFields[field] && String(item[field]).toLowerCase().includes(filter))) : data; const rows = filtered.map(item =><tr> ${settings.dataFields.name ?{escapeHTML(item.name)}</td>` : ''} {settings.dataFields.url ?<td><a href="${escapeHTML(item.url)}" target="_blank">${escapeHTML(item.url)}</a></td>: ''} {settings.dataFields.description ? `<td>{escapeHTML(item.description)}: ''} ${settings.dataFields.html ?📄 보기: ''} ${settings.dataFields.category ?{escapeHTML(item.category)}</td>` : ''} {settings.dataFields.quality ?<td>${escapeHTML(item.quality.toString())}</td>: ''} {settings.dataFields.language ? `<td>{escapeHTML(item.language)}: ''} ${settings.dataFields.wordCount ?{escapeHTML(item.wordCount.toString())}</td>` : ''} {settings.dataFields.metaTags ?<td>${escapeHTML(JSON.stringify(item.metaTags))}</td>: ''} {settings.dataFields.headings ? `<td>{escapeHTML(item.headings.join(', '))}: ''} </tr>).join(''); ('#data-table-body').html(rows); ('.view-html').off('click').on('click', function() { ('#modal-html-content').text(decodeURIComponent((this).data('html'))); ('#html-modal').show(); }); if (settings.dataVisualization) renderChart(); } function renderChart() { if (!document.getElementById('data-chart').getContext) return; const ctx = document.getElementById('data-chart').getContext('2d'); if (window.dataChartInstance) window.dataChartInstance.destroy(); const categories = {}; data.forEach(item => { const cat = item.category || '기타'; categories[cat] = (categories[cat] || 0) + 1; }); const chartData = { labels: Object.keys(categories), datasets: [{ label: '카테고리별 데이터 수', data: Object.values(categories), backgroundColor: generateColors(Object.keys(categories).length) }] }; window.dataChartInstance = new Chart(ctx, { type: 'pie', data: chartData, options: { responsive: true } }); } function generateColors(count) { return Array.from({length: count}, (_, i) => `hsl({i * (360 / count)}, 70%, 50%)); } function escapeHTML(str) { return String(str).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'"); } function downloadCSV() { if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; } const headers = Object.keys(settings.dataFields).filter(f => settings.dataFields[f]).map(f => getFieldName(f)); const csv = [headers.join(",")].concat(data.map(item => Object.keys(settings.dataFields).filter(f => settings.dataFields[f]).map(f =>"{encodeURIComponent(csv)}; link.download = "data.csv"; document.body.appendChild(link); link.click(); document.body.removeChild(link); log('CSV 파일이 다운로드되었습니다.', 'success'); } function downloadJSON() { if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; } const json = JSON.stringify(data, null, 2); const blob = new Blob([json], { type: "application/json" }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); link.download = "data.json"; document.body.appendChild(link); link.click(); document.body.removeChild(link); log('JSON 파일이 다운로드되었습니다.', 'success'); } function downloadXML() { if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; } let xml = '<?xml version="1.0" encoding="UTF-8"?><data>'; data.forEach(item => { xml += '<entry>'; if (settings.dataFields.name) xml +={escapeXML(item.url || '')}; if (settings.dataFields.description) xml +={item.html || ''}]]>; if (settings.dataFields.category) xml +={escapeXML(item.quality)}; if (settings.dataFields.language) xml +={escapeXML(item.wordCount)}; if (settings.dataFields.metaTags) xml +={escapeXML(item.headings.join('; '))}; xml += '</entry>'; }); xml += '</data>'; const blob = new Blob([xml], { type: "application/xml" }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); link.download = "data.xml"; document.body.appendChild(link); link.click(); document.body.removeChild(link); log('XML 파일이 다운로드되었습니다.', 'success'); } function downloadXLS() { if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; } const sheet = XLSX.utils.json_to_sheet(data.map(item => { const entry = {}; if (settings.dataFields.name) entry['사이트 이름'] = item.name; if (settings.dataFields.url) entry['주소'] = item.url; if (settings.dataFields.description) entry['설명'] = item.description; if (settings.dataFields.html) entry['📄 HTML 내용'] = item.html; if (settings.dataFields.category) entry['분류'] = item.category; if (settings.dataFields.quality) entry['품질점수'] = item.quality; if (settings.dataFields.language) entry['언어'] = item.language; if (settings.dataFields.wordCount) entry['단어 수'] = item.wordCount; if (settings.dataFields.metaTags) entry['메타 태그'] = JSON.stringify(item.metaTags); if (settings.dataFields.headings) entry['헤딩'] = item.headings.join('; '); return entry; })); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, sheet, "Data"); XLSX.writeFile(wb, "data.xlsx"); log('XLS 파일이 다운로드되었습니다.', 'success'); } function escapeCSV(str) { return String(str).replace(/"/g, '""'); } function filterTable(keyword) { $('#data-table-body tr').each(function() { $(this).toggle($(this).text().toLowerCase().includes(keyword.toLowerCase())); }); } function saveSettings() { settings.delay = parseInt($('#delay-input').val(), 10) || defaultSettings.delay; settings.delayRandom = parseInt($('#delay-random-input').val(), 10) || defaultSettings.delayRandom; settings.maxPages = parseInt($('#max-pages-input').val(), 10) || defaultSettings.maxPages; const selectedUA = parseInt($('#user-agent-select').val(), 10); settings.userAgentIndex = isNaN(selectedUA) ? 0 : selectedUA; settings.contentTypes.include = $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(ct => ct) || defaultSettings.contentTypes.include; settings.contentTypes.exclude = $('#content-exclude-input').val().split(',').map(ct => ct.trim()).filter(ct => ct) || defaultSettings.contentTypes.exclude; settings.timeout = parseInt($('#timeout-input').val(), 10) || defaultSettings.timeout; settings.includeKeywords = $('#include-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw) || defaultSettings.includeKeywords; settings.excludeKeywords = $('#exclude-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw) || defaultSettings.excludeKeywords; settings.regexInclude = $('#regex-include-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx) || defaultSettings.regexInclude; settings.regexExclude = $('#regex-exclude-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx) || defaultSettings.regexExclude; settings.language = $('#language-select').val() || []; settings.minWordCount = parseInt($('#min-word-count-input').val(), 10) || defaultSettings.minWordCount; settings.maxWordCount = parseInt($('#max-word-count-input').val(), 10) || defaultSettings.maxWordCount; settings.maxDepth = parseInt($('#max-depth-input').val(), 10) || defaultSettings.maxDepth; settings.proxy = $('#proxy-input').val().trim() || defaultSettings.proxy; ['name','url','description','html','category','quality','language','wordCount','metaTags','headings'].forEach(field => { settings.dataFields[field] = $(#field-{field}`).is(':checked'); }); settings.retryAttempts = parseInt(('#retry-attempts-input').val(), 10) || defaultSettings.retryAttempts; settings.retryDelay = parseInt(('#retry-delay-input').val(), 10) || defaultSettings.retryDelay; settings.schedule.enabled = ('#schedule-enabled').is(':checked'); settings.schedule.startTime = ('#start-time-input').val() || defaultSettings.schedule.startTime; settings.schedule.interval = parseInt(('#interval-input').val(), 10) ? parseInt(('#interval-input').val(), 10) * 60000 : defaultSettings.schedule.interval; Object.keys(settings.searchEngines).forEach(engine => { settings.searchEngines[engine] = (#engine-${engine}).is(':checked'); }); settings.blacklist = ('#blacklist-input').val().split('\n').map(d => d.trim()).filter(d => d) || defaultSettings.blacklist; settings.whitelist = ('#whitelist-input').val().split('\n').map(d => d.trim()).filter(d => d) || defaultSettings.whitelist; settings.respectRobots = ('#respect-robots-checkbox').is(':checked'); settings.maxConcurrentRequests = parseInt(('#max-concurrent-requests-input').val(), 10) || defaultSettings.maxConcurrentRequests; settings.dataVisualization = ('#data-visualization-checkbox').is(':checked'); settings.exportFormats.csv = ('#export-csv').is(':checked'); settings.exportFormats.json = ('#export-json').is(':checked'); settings.exportFormats.xml = ('#export-xml').is(':checked'); settings.exportFormats.xls = ('#export-xls').is(':checked'); settings.advancedLogging = ('#advanced-logging-checkbox').is(':checked'); settings.enableCaptchas = ('#enable-captchas-checkbox').is(':checked'); const customKeywords = parseCustomKeywords(('#custom-keywords-input').val()); settings.customKeywords = customKeywords; GM_setValue('delay', settings.delay); GM_setValue('delayRandom', settings.delayRandom); GM_setValue('maxPages', settings.maxPages); GM_setValue('userAgentIndex', settings.userAgentIndex); GM_setValue('contentTypes', settings.contentTypes); GM_setValue('timeout', settings.timeout); GM_setValue('includeKeywords', settings.includeKeywords); GM_setValue('excludeKeywords', settings.excludeKeywords); GM_setValue('regexInclude', settings.regexInclude); GM_setValue('regexExclude', settings.regexExclude); GM_setValue('language', settings.language); GM_setValue('minWordCount', settings.minWordCount); GM_setValue('maxWordCount', settings.maxWordCount); GM_setValue('maxDepth', settings.maxDepth); GM_setValue('proxy', settings.proxy); GM_setValue('dataFields', settings.dataFields); GM_setValue('retryAttempts', settings.retryAttempts); GM_setValue('retryDelay', settings.retryDelay); GM_setValue('schedule', settings.schedule); GM_setValue('searchEngines', settings.searchEngines); GM_setValue('blacklist', settings.blacklist); GM_setValue('whitelist', settings.whitelist); GM_setValue('respectRobots', settings.respectRobots); GM_setValue('maxConcurrentRequests', settings.maxConcurrentRequests); GM_setValue('dataVisualization', settings.dataVisualization); GM_setValue('exportFormats', settings.exportFormats); GM_setValue('advancedLogging', settings.advancedLogging); GM_setValue('enableCaptchas', settings.enableCaptchas); GM_setValue('customKeywords', settings.customKeywords); totalPages = settings.maxPages; GM_setValue('totalPages', totalPages); log('설정이 저장되었습니다.', 'success'); if (settings.schedule.enabled) scheduleCrawling(); updateDataUI(); if (settings.dataVisualization) renderChart(); } function resetSettings() { if (confirm("기본 설정으로 초기화하시겠습니까?")) { Object.keys(defaultSettings).forEach(key => GM_deleteValue(key)); Object.assign(settings, defaultSettings); ['userAgents','userAgentIndex','contentTypes','dataFields','exportFormats','advancedLogging','enableCaptchas'].forEach(key => GM_setValue(key, settings[key])); GM_setValue('customKeywords', {}); GM_setValue('logs', []); GM_setValue('data', []); GM_setValue('visitedUrls', []); GM_setValue('previousCrawls', []); ('#custom-keywords-input').val(''); updateUserAgentSelect(); updateDataUI(); updateLogsUI(); ('#previous-crawls-body').empty(); if (settings.dataVisualization) renderChart(); log('기본 설정으로 초기화되었습니다.', 'success'); } } async function startCrawling() { if (isCrawling) { log('이미 크롤링이 진행 중입니다.', 'warning'); return; } if (!Object.values(settings.searchEngines).some(v => v)) { log('사용할 검색 엔진이 하나도 선택되지 않았습니다.', 'error'); return; } isCrawling = true; GM_setValue('isCrawling', isCrawling); log('크롤링을 시작합니다.', 'info'); setButtonStates(); currentPage = GM_getValue('currentPage', 1); totalPages = settings.maxPages; GM_setValue('totalPages', totalPages); currentDepth = GM_getValue('currentDepth', 0); if (!previousCrawls.some(crawl => crawl.status === '진행 중')) { previousCrawls.push({ date: new Date().toLocaleString(), count: 0, status: '진행 중' }); GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } await crawlPage(currentDepth); } function stopCrawling() { if (!isCrawling) { log('크롤링이 진행 중이지 않습니다.', 'warning'); return; } isCrawling = false; GM_setValue('isCrawling', isCrawling); log('크롤링이 중지되었습니다.', 'warning'); finalizeCrawl('중지됨'); setButtonStates(); } function setButtonStates() { ('#start-crawling').prop('disabled', isCrawling); ('#stop-crawling').prop('disabled', !isCrawling); } async function crawlPage(depth) { try { while (isCrawling && currentPage <= totalPages && depth <= settings.maxDepth) { log(페이지 ${currentPage} 크롤링 시작. (깊이: ${depth}), 'info'); if (settings.respectRobots && !(await checkRobotsTxt(window.location.href))) { log(robots.txt에 의해 크롤링이 금지된 페이지: ${window.location.href}, 'warning'); break; } const results = extractSearchResults(); if (!results) { log('검색 결과를 찾을 수 없습니다.', 'error'); break; } results.forEach(result => { if (!isCrawling) return; const { name, url, description } = result; if (visitedUrls.includes(url) || !isValidURL(url) || (settings.whitelist.length && !settings.whitelist.some(d => url.includes(d))) || settings.blacklist.some(d => url.includes(d)) || (settings.includeKeywords.length && !settings.includeKeywords.some(k => url.includes(k) || description.includes(k))) || (settings.excludeKeywords.some(k => url.includes(k) || description.includes(k))) || (settings.regexInclude.length && !settings.regexInclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description))) || (settings.regexExclude.length && settings.regexExclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description)))) { return; } requestQueue.push({ url, depth }); if (activeRequests >= settings.maxConcurrentRequests) return; }); if (previousCrawls.length && previousCrawls[previousCrawls.length -1].status === '진행 중') { previousCrawls[previousCrawls.length -1].count += results.length; GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } await processQueue(); progress = (currentPage / totalPages) * 100; GM_setValue('progress', progress); updateLogsUI(); if (currentPage < totalPages && isCrawling) { const nextUrl = getNextPageUrl(); if (nextUrl) { currentPage++; GM_setValue('currentPage', currentPage); depth++; GM_setValue('currentDepth', depth); log(다음 페이지로 이동: ${currentPage}, 'info'); await dynamicDelay(); window.location.href = nextUrl; return; } else { log('다음 페이지 링크를 찾을 수 없습니다.', 'error'); break; } } else { break; } } if (isCrawling && currentPage > totalPages && activeRequests === 0 && requestQueue.length === 0) { log('크롤링이 완료되었습니다.', 'success'); finalizeCrawl('완료됨'); isCrawling = false; GM_setValue('isCrawling', isCrawling); setButtonStates(); GM_setValue('progress', 100); ('.status-text').text(진행률: 100% (${currentPage}/${totalPages} 페이지)); } } catch (error) { log(크롤링 중 오류 발생: ${error.message}, 'error'); finalizeCrawl('오류 발생'); } } async function processQueue() { while (requestQueue.length && activeRequests < settings.maxConcurrentRequests && isCrawling) { const { url, depth } = requestQueue.shift(); activeRequests++; handleRequest(url, depth).finally(() => { activeRequests--; processQueue(); }); } } async function handleRequest(url, depth) { if (!isCrawling || visitedUrls.includes(url)) return; try { const response = await fetchWithRetry(url, settings.retryAttempts, settings.retryDelay); const contentType = getContentType(response); if (!isContentTypeAllowed(contentType)) { log(허용되지 않은 콘텐츠 유형(${contentType})의 URL: ${url}, 'warning'); return; } const parser = new DOMParser(); const doc = parser.parseFromString(response.responseText, 'text/html'); const text = doc.body ? doc.body.innerText : ''; if (settings.enableCaptchas && detectCaptcha(doc)) { await handleCaptcha(url); return; } const wc = text.split(/\s+/).length; if (wc < settings.minWordCount || wc > settings.maxWordCount) { log(단어 수 조건에 맞지 않는 URL: ${url} (${wc} 단어), 'warning'); return; } const quality = evaluateContentQuality(text); const lang = doc.documentElement.lang || 'unknown'; const category = classifyKeywords(text); const meta = extractMetaTags(doc); const headings = extractHeadings(doc); if (category !== '기타' || quality > 100) { const entry = { url, quality, language: lang, wordCount: wc, category }; if (settings.dataFields.name) entry.name = extractName(doc) || 'No Title'; if (settings.dataFields.description) entry.description = extractDescription(doc) || 'No Description'; if (settings.dataFields.html) entry.html = response.responseText; if (settings.dataFields.metaTags) entry.metaTags = meta; if (settings.dataFields.headings) entry.headings = headings; data.push(entry); GM_setValue('data', data); updateDataUI(); log(데이터 수집 완료: ${url}, 'success'); if (settings.dataVisualization) renderChart(); } if (depth < settings.maxDepth) { const links = extractLinks(doc); links.forEach(link => { if (!visitedUrls.includes(link) && isValidURL(link)) requestQueue.push({ url: link, depth: depth + 1 }); }); } } catch (error) { log(데이터 수집 실패: ${url} - ${error.message}, 'error'); } visitedUrls.push(url); GM_setValue('visitedUrls', visitedUrls); } async function fetchWithRetry(url, attempts, delayMs) { for (let i = 1; i <= attempts; i++) { try { return await GM_xmlhttpRequestPromise(url); } catch (e) { log(요청 실패 (${i}/${attempts}): ${url} - ${e.message}, 'warning'); if (i < attempts) await delay(delayMs); } } throw new Error('All retry attempts failed'); } function isAnySearchEngineEnabled() { return Object.values(settings.searchEngines).some(v => v); } function classifyKeywords(text) { const kws = combineKeywords(); for (const [cat, keywords] of Object.entries(kws)) { if (keywords.some(kw => text.includes(kw))) return cat; } return '기타'; } function extractSearchResults() { let results = []; const host = window.location.hostname; if (host.includes('google.com') && settings.searchEngines.google) { document.querySelectorAll('div.g, div[data-hveid]').forEach(item => { const name = item.querySelector('h3')?.innerText.trim() || 'No Title'; const url = item.querySelector('a')?.href || ''; const desc = item.querySelector('.aCOpRe, .VwiC3b')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); } else if (host.includes('bing.com') && settings.searchEngines.bing) { document.querySelectorAll('.b_algo').forEach(item => { const name = item.querySelector('h2')?.innerText.trim() || 'No Title'; const url = item.querySelector('h2 a')?.href || ''; const desc = item.querySelector('.b_caption p')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); } // Add similar blocks for other search engines... log(검색 엔진(${host.includes('google.com') ? 'Google' : host.includes('bing.com') ? 'Bing' : '기타'})에서 검색 결과 ${results.length}개 추출됨., 'info'); return results.length ? results : null; } function getNextPageUrl() { const host = window.location.hostname; let next = null; if (host.includes('google.com')) next = document.querySelector('a#pnnext')?.href; else if (host.includes('bing.com')) next = document.querySelector('a.sb_pagN')?.href; // Add similar blocks for other search engines... log(다음 페이지 URL: ${next || '없음'}, 'info'); return next; } function GM_xmlhttpRequestPromise(url) { return new Promise((resolve, reject) => { let reqUrl = settings.proxy ?${settings.proxy}?url=${encodeURIComponent(url)}: url; GM_xmlhttpRequest({ method: "GET", url: reqUrl, headers: { 'User-Agent': getCurrentUserAgent(), 'Accept': '/' }, timeout: settings.timeout, onload: res => (res.status >= 200 && res.status < 300) ? resolve(res) : reject(new Error(HTTP ${res.status})), onerror: () => reject(new Error('Request failed')), ontimeout: () => reject(new Error('Request timed out')) }); }); } function getCurrentUserAgent() { return settings.userAgents[settings.userAgentIndex % settings.userAgents.length]; } function getContentType(res) { return res.responseHeaders.split('\n').find(h => h.toLowerCase().startsWith('content-type'))?.split(':')[1].trim().split(';')[0] || ''; } function isContentTypeAllowed(ct) { if (settings.contentTypes.include.length && !settings.contentTypes.include.includes(ct)) return false; if (settings.contentTypes.exclude.includes(ct)) return false; return true; } function evaluateContentQuality(text) { return text.split(/\s+/).length; } function extractMetaTags(doc) { const metas = {}; doc.querySelectorAll('meta[name], meta[property]').forEach(meta => { const name = meta.getAttribute('name') || meta.getAttribute('property'); const content = meta.getAttribute('content'); if (name && content) metas[name.toLowerCase()] = content; }); return metas; } function extractHeadings(doc) { const hs = []; for (let i = 1; i <= 6; i++) doc.querySelectorAll(h${i}).forEach(h => hs.push(h.innerText.trim())); return hs; } function isValidURL(url) { try { new URL(url); return true; } catch { return false; } } async function checkRobotsTxt(url) { try { const parsed = new URL(url); const robots =${parsed.origin}/robots.txt; const res = await GM_xmlhttpRequestPromise(robots); return new RobotsParser(robots, res.responseText).isAllowed(url, getCurrentUserAgent()); } catch { log('robots.txt 확인 실패, 기본 허용.', 'warning'); return true; } } class RobotsParser { constructor(url, text) { this.rules = []; this.parse(text); } parse(text) { let agents = []; text.split('\n').forEach(line => { const [key, ...val] = line.split(':').map(s => s.trim()); if (!key) return; if (key.toLowerCase() === 'user-agent') { agents = [val.join(':').toLowerCase()]; } else if (['disallow','allow'].includes(key.toLowerCase())) this.rules.push({ agents, type: key.toLowerCase(), path: val.join(':') }); }); } isAllowed(url, ua) { const path = new URL(url).pathname; let allowed = true; this.rules.forEach(rule => { if (rule.agents.includes('*') || rule.agents.includes(ua.toLowerCase())) { if (rule.type === 'disallow' && path.startsWith(rule.path)) allowed = false; if (rule.type === 'allow' && path.startsWith(rule.path)) allowed = true; } }); return allowed; } } function delay(ms) { return new Promise(res => setTimeout(res, ms)); } function formatCustomKeywords(ck) { return Object.entries(ck).map(([c, kws]) =>${c}:${kws.join(',')}).join('\n'); } function getFieldName(field) { return { name: '사이트 이름', url: '주소', description: '설명', html: '📄 HTML 내용', category: '분류', quality: '품질점수', language: '언어', wordCount: '단어 수', metaTags: '메타 태그', headings: '헤딩' }[field]; } function capitalize(str) { return str.charAt(0).toUpperCase() + str.slice(1); } async function handleCaptcha(url) { log(CAPTCHA 감지됨: ${url}, 'warning'); // CAPTCHA 처리 로직 추가 (제한적) // 예: CAPTCHA 해결을 위한 별도의 서비스 연동 또는 수동 해결 유도 } function extractName(doc) { return doc.querySelector('title')?.innerText || 'No Title'; } function extractDescription(doc) { return doc.querySelector('meta[name="description"]')?.getAttribute('content') || 'No Description'; } function extractLinks(doc) { return Array.from(doc.querySelectorAll('a[href]')).map(a => a.href).filter(isValidURL); } function escapeXML(str) { return String(str).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'"); } function init() { createUI(); if (isCrawling) { log('크롤링 재개됨.', 'info'); setButtonStates(); crawlPage(currentDepth); } else log('크롤러 초기화됨.', 'info'); } window.addEventListener('load', init); function makeDraggable() { const container = document.getElementById('crawler-container'); const header = document.getElementById('crawler-header'); let isDragging = false; let startX, startY, initialX, initialY; header.onmousedown = function(e) { isDragging = true; startX = e.clientX; startY = e.clientY; initialX = container.offsetLeft; initialY = container.offsetTop; document.onmousemove = drag; document.onmouseup = stopDrag; return false; }; function drag(e) { if (isDragging) { let dx = e.clientX - startX; let dy = e.clientY - startY; container.style.left = (initialX + dx) + 'px'; container.style.top = (initialY + dy) + 'px'; } } function stopDrag() { isDragging = false; document.onmousemove = null; document.onmouseup = null; } } function evaluateContentQuality(text) { return text.split(/\s+/).length; } async function scheduleCrawling() { const now = new Date(); const [startHour, startMinute] = settings.schedule.startTime.split(':').map(Number); const startTime = new Date(now.getFullYear(), now.getMonth(), now.getDate(), startHour, startMinute, 0); if (startTime < now) startTime.setDate(startTime.getDate() + 1); const initialDelay = startTime - now; setTimeout(() => { startCrawling(); setInterval(startCrawling, settings.schedule.interval); }, initialDelay); log(크롤링 스케줄이 설정되었습니다. 시작 시간: ${startTime.toLocaleString()}, 간격: ${settings.schedule.interval / 60000}분, 'info'); } function finalizeCrawl(status) { if (previousCrawls.length) { previousCrawls[previousCrawls.length -1].status = status; GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } } function updatePreviousCrawlsUI() { ('#previous-crawls-body').html(previousCrawls.map(crawl => ` <tr> <td>{crawl.date} {crawl.status}).join('')); } function updatePreviousCrawlsUI() { $('#previous-crawls-body').html(previousCrawls.map(crawl =>{crawl.count} ${crawl.status} `).join('')); } // 추가적인 기능이나 수정이 필요한 부분은 이곳에 구현하십시오. })();
검색 엔진별 크롤링 로직의 일관성 유지 현재 extractSearchResults 함수 내에서 Google과 Bing에 대한 크롤링 로직이 구현되어 있으나, 다른 검색 엔진에 대한 로직은 주석으로 처리되어 있습니다. 이는 스크립트의 완전성을 저해하며, 유지보수를 어렵게 만듭니다.개선 방안:
검색 엔진별 핸들러 객체 , 에러 핸들링 개선 현재 크롤링 중 에러가 발생하면 로그를 남기지만, 에러의 원인을 보다 상세히 파악할 수 있는 방법이 부족합니다. 또한, 네트워크 요청 실패 시 모든 재시도 시도가 완료되었을 때의 처리가 부족합니다.
개선 방안:
상세 에러 로그: 에러 발생 시 더 많은 정보를 로그에 포함시킵니다. 재시도 로직 강화: 재시도 실패 시 크롤링을 중단하거나, 특정 동작을 추가로 수행합니다. , 비동기 처리의 효율성 향상 현재 crawlPage 함수는 페이지를 크롤링하면서 window.location.href를 변경하여 다음 페이지로 이동하고 있습니다. 이는 Tampermonkey 스크립트의 동작 방식과 충돌할 수 있으며, 스크립트의 연속성을 방해할 수 있습니다.
개선 방안:
브라우저 내비게이션 대신 AJAX 요청 사용: 페이지 전환 없이 배경에서 다음 페이지의 내용을 가져와 처리할 수 있습니다. 이를 통해 스크립트의 실행이 중단되지 않도록 합니다. 싱글 페이지 앱(SPA) 방식 고려: 만약 검색 엔진이 SPA 방식이라면, URL 변경 시 이벤트를 감지하여 크롤링을 계속할 수 있습니다. , 로딩 상태 표시: 대량의 데이터를 처리할 때 사용자에게 진행 상태를 명확히 알려주는 로딩 스피너나 애니메이션을 추가합니다. 에러 메시지 가시성 향상: 중요한 에러 메시지는 사용자에게 더 눈에 띄게 표시하여 즉각적으로 대응할 수 있도록 합니다. 설정 검증: 사용자 입력값에 대한 유효성을 검사하고, 잘못된 입력 시 경고를 표시하여 오류를 예방합니다. , 중복 URL 검출 최적화: 현재 visitedUrls는 배열로 관리되고 있는데, 대량의 URL을 처리할 경우 성능 저하가 발생할 수 있습니다. 이를 Set 자료구조로 변경하여 O(1)의 검색 시간을 확보할 수 있습니다. 자 이제 이것들을 줄 테니깐 위에 코드를 이걸를 바탕으로 추가하거나 개선 해 . 일단 최대한 니가 쓸 수 있는데 까지는 아무말없이 써봐. 그냥 코드만 써. ㅇㅇ 그리고 니가 써도 되는 용량이 최대가 될때 파트 1 으로 끝어 알겠지? 최대한 길게 써. 최대한 길게!! 그리고 ㅈㄴ 오래길게 생각해봐 그게 맞는지 아닌지 ㅈㄴ 오래 생각해봐 ㅈㄴ 짫게 쓰면 사용작가 지랄할 지 안할지
- Assistant
// ==UserScript== // @name Advanced Web Crawler Enhanced Pro Optimized v6.0.4 // @namespace http://tampermonkey.net/ // @version 6.0.4 // @description Optimized and enhanced version of Advanced Web Crawler Enhanced Pro with improved error handling, asynchronous processing, and performance optimizations // @author Your Name // @match https://www.google.com/search* // @match https://www.bing.com/search* // @match https://search.yahoo.com/search* // @match https://duckduckgo.com/* // @match https://www.baidu.com/s* // @match https://www.yandex.com/search/* // @match https://search.naver.com/search* // @match https://search.daum.net/search* // @match https://www.ask.com/* // @match https://www.ecosia.org/search* // @match https://www.startpage.com/* // @match https://searx.me/* // @match https://www.dogpile.com/* // @match https://www.bing.com/images* // @grant GM_xmlhttpRequest // @grant GM_setValue // @grant GM_getValue // @grant GM_deleteValue // @connect * // @require https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js // @require https://cdn.jsdelivr.net/npm/chart.js // @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js // ==/UserScript== (function() { 'use strict'; const defaultSettings = { delay: 2000, delayRandom: 500, maxPages: 3, userAgents: [ 'AdvancedWebCrawler/6.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)', 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15', 'Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15', ], userAgentIndex: 0, blacklist: [], whitelist: [], contentTypes: { include: ['text/html'], exclude: ['application/pdf', 'image/jpeg', 'image/png', 'video/mp4'] }, timeout: 8000, includeKeywords: [], excludeKeywords: [], language: [], minWordCount: 100, maxWordCount: 10000, maxDepth: 2, proxy: '', regexInclude: [], regexExclude: [], dataFields: { name: true, url: true, description: true, html: true, category: true, quality: true, language: true, wordCount: true, metaTags: false, headings: false }, schedule: { enabled: false, startTime: '00:00', interval: 3600000 }, searchEngines: { google: true, bing: true, yahoo: true, duckduckgo: true, baidu: true, yandex: true, naver: true, daum: true, ask: true, ecosia: true, startpage: true, searx: true, dogpile: true, bingimages: true }, retryAttempts: 3, retryDelay: 2000, respectRobots: true, maxConcurrentRequests: 3, dataVisualization: true, exportFormats: { csv: true, json: true, xml: true, xls: true }, advancedLogging: true, enableCaptchas: false }; const settings = Object.assign({}, defaultSettings, { delay: parseInt(GM_getValue('delay', defaultSettings.delay), 10), delayRandom: parseInt(GM_getValue('delayRandom', defaultSettings.delayRandom), 10), maxPages: parseInt(GM_getValue('maxPages', defaultSettings.maxPages), 10), userAgents: GM_getValue('userAgents', defaultSettings.userAgents), userAgentIndex: parseInt(GM_getValue('userAgentIndex', defaultSettings.userAgentIndex), 10), blacklist: GM_getValue('blacklist', defaultSettings.blacklist), whitelist: GM_getValue('whitelist', defaultSettings.whitelist), contentTypes: GM_getValue('contentTypes', defaultSettings.contentTypes), timeout: parseInt(GM_getValue('timeout', defaultSettings.timeout), 10), includeKeywords: GM_getValue('includeKeywords', defaultSettings.includeKeywords), excludeKeywords: GM_getValue('excludeKeywords', defaultSettings.excludeKeywords), regexInclude: GM_getValue('regexInclude', defaultSettings.regexInclude), regexExclude: GM_getValue('regexExclude', defaultSettings.regexExclude), language: GM_getValue('language', defaultSettings.language), minWordCount: parseInt(GM_getValue('minWordCount', defaultSettings.minWordCount), 10), maxWordCount: parseInt(GM_getValue('maxWordCount', defaultSettings.maxWordCount), 10), maxDepth: parseInt(GM_getValue('maxDepth', defaultSettings.maxDepth), 10), proxy: GM_getValue('proxy', defaultSettings.proxy), dataFields: GM_getValue('dataFields', defaultSettings.dataFields), schedule: GM_getValue('schedule', defaultSettings.schedule), searchEngines: GM_getValue('searchEngines', defaultSettings.searchEngines), retryAttempts: parseInt(GM_getValue('retryAttempts', defaultSettings.retryAttempts), 10), retryDelay: parseInt(GM_getValue('retryDelay', defaultSettings.retryDelay), 10), respectRobots: GM_getValue('respectRobots', defaultSettings.respectRobots), maxConcurrentRequests: parseInt(GM_getValue('maxConcurrentRequests', defaultSettings.maxConcurrentRequests), 10), dataVisualization: GM_getValue('dataVisualization', defaultSettings.dataVisualization), exportFormats: GM_getValue('exportFormats', defaultSettings.exportFormats), advancedLogging: GM_getValue('advancedLogging', defaultSettings.advancedLogging), enableCaptchas: GM_getValue('enableCaptchas', defaultSettings.enableCaptchas), customKeywords: GM_getValue('customKeywords', {}) }); let visitedUrls = new Set(GM_getValue('visitedUrls', [])); let logs = GM_getValue('logs', []); let data = GM_getValue('data', []); let isCrawling = GM_getValue('isCrawling', false); let currentPage = parseInt(GM_getValue('currentPage', 1), 10); let totalPages = parseInt(GM_getValue('totalPages', settings.maxPages), 10); let progress = parseFloat(GM_getValue('progress', 0)); let currentDepth = parseInt(GM_getValue('currentDepth', 0), 10); let activeRequests = 0; let requestQueue = []; let previousCrawls = GM_getValue('previousCrawls', []); const searchEngineHandlers = { google: { extractResults: function(doc) { const results = []; doc.querySelectorAll('div.g, div[data-hveid]').forEach(item => { const name = item.querySelector('h3')?.innerText.trim() || 'No Title'; const url = item.querySelector('a')?.href || ''; const desc = item.querySelector('.aCOpRe, .VwiC3b')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: function(doc) { return doc.querySelector('a#pnnext')?.href || null; } }, bing: { extractResults: function(doc) { const results = []; doc.querySelectorAll('.b_algo').forEach(item => { const name = item.querySelector('h2')?.innerText.trim() || 'No Title'; const url = item.querySelector('h2 a')?.href || ''; const desc = item.querySelector('.b_caption p')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: function(doc) { return doc.querySelector('a.sb_pagN')?.href || null; } }, yahoo: { extractResults: function(doc) { const results = []; doc.querySelectorAll('.Sr').forEach(item => { const name = item.querySelector('h3')?.innerText.trim() || 'No Title'; const url = item.querySelector('a')?.href || ''; const desc = item.querySelector('.compText a')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: function(doc) { return doc.querySelector('a.next')?.href || null; } }, // Add similar handlers for other search engines... duckduckgo: { extractResults: function(doc) { const results = []; doc.querySelectorAll('.result').forEach(item => { const name = item.querySelector('.result__title')?.innerText.trim() || 'No Title'; const url = item.querySelector('.result__url')?.href || ''; const desc = item.querySelector('.result__snippet')?.innerText.trim() || 'No Description'; if (isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: function(doc) { return doc.querySelector('a.result--more__btn')?.href || null; } }, // Implement other search engines as needed }; function createUI() { const style = ` #crawler-container { position: fixed; top: 20px; right: 20px; width: 400px; height: 500px; background: #fff; border: 1px solid #ccc; z-index: 10000; font-family: Arial, sans-serif; box-shadow: 0 4px 8px rgba(0,0,0,0.2); overflow: auto; resize: both; border-radius: 8px; min-width: 300px; min-height: 300px; } #crawler-header { display: flex; justify-content: space-between; align-items: center; background: #f1f1f1; border-bottom: 1px solid #ccc; padding: 5px 10px; cursor: move; } #crawler-minimize { background: none; border: none; font-size: 18px; cursor: pointer; } #crawler-tabs { display: flex; background: #f1f1f1; border-bottom: 1px solid #ccc; } .crawler-tab { flex: 1; padding: 10px; text-align: center; cursor: pointer; font-weight: bold; } .crawler-tab.active { background: #ddd; } #crawler-content { padding: 10px; overflow: auto; height: calc(100% - 50px); } table { width: 100%; border-collapse: collapse; } th, td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; } th { background-color: #f2f2f2; cursor: pointer; } tr:hover { background-color: #f5f5f5; } .progress-bar { width: 100%; background-color: #f3f3f3; border: 1px solid #ccc; border-radius: 5px; overflow: hidden; margin-bottom: 10px; } .progress-bar-fill { height: 20px; background-color: #4caf50; width: 0%; transition: width 0.2s; } .modal { display: none; position: fixed; z-index: 10001; left: 0; top: 0; width: 100%; height: 100%; overflow: auto; background-color: rgba(0,0,0,0.4); } .modal-content { background-color: #fff; margin: 5% auto; padding: 20px; border: 1px solid #888; width: 80%; max-height: 80%; overflow-y: auto; border-radius: 8px; } .close { color: #aaa; float: right; font-size: 28px; font-weight: bold; cursor: pointer; } .control-buttons { margin: 10px 0; } .control-buttons button { padding: 8px 12px; margin-right: 8px; cursor: pointer; border: none; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px; } .control-buttons button:disabled { background-color: #a5d6a7; cursor: not-allowed; } .status-text { margin-top: 10px; font-weight: bold; font-size: 14px; } .settings-section { margin-bottom: 15px; } .settings-section label { display: block; margin-bottom: 5px; font-weight: bold; } .settings-section input[type="text"], .settings-section input[type="number"], .settings-section textarea, .settings-section select { width: 100%; padding: 5px; box-sizing: border-box; border: 1px solid #ccc; border-radius: 4px; font-size: 14px; } .settings-section .checkbox-group { display: flex; flex-wrap: wrap; } .settings-section .checkbox-group label { margin-right: 10px; font-size: 14px; } .settings-section .radio-group { display: flex; flex-wrap: wrap; } .settings-section .radio-group label { margin-right: 10px; font-size: 14px; } .tooltip { position: relative; display: inline-block; cursor: help; font-size: 14px; } .tooltip .tooltiptext { visibility: hidden; width: 220px; background-color: #555; color: #fff; text-align: center; border-radius: 6px; padding: 5px; position: absolute; z-index: 1; bottom: 125%; left: 50%; margin-left: -110px; opacity: 0; transition: opacity 0.3s; } .tooltip:hover .tooltiptext { visibility: visible; opacity: 1; } #crawler-container::-webkit-resizer { background: #f1f1f1; border: none; } #data-visualization { margin-top: 20px; } #data-chart { width: 100%; height: 300px; } #modal-html-content { white-space: pre-wrap; word-wrap: break-word; background-color: #f9f9f9; padding: 10px; border-radius: 4px; } #crawler-tabs .crawler-tab:last-child { flex: 1; } #previous-crawls { margin-top: 20px; } #previous-crawls table { width: 100%; } #previous-crawls th, #previous-crawls td { padding: 5px; } #reset-data { background-color: #f44336; } /* Loading Spinner */ .spinner { border: 4px solid #f3f3f3; border-top: 4px solid #3498db; border-radius: 50%; width: 24px; height: 24px; animation: spin 2s linear infinite; display: inline-block; vertical-align: middle; } @keyframes spin { 0% { transform: rotate(0deg); } 100% { transform: rotate(360deg); } } /* Error Message Styling */ .error-message { color: red; font-weight: bold; margin-top: 10px; } /* Success Message Styling */ .success-message { color: green; font-weight: bold; margin-top: 10px; } `; $('head').append(`<style>${style}</style>`); const container = $(` <div id="crawler-container"> <div id="crawler-header"> <div id="crawler-tabs"> <div class="crawler-tab ${currentTab() === 'logs' ? 'active' : ''}" data-tab="logs" title="로그 보기">📜 로그</div> <div class="crawler-tab ${currentTab() === 'data' ? 'active' : ''}" data-tab="data" title="수집된 데이터 보기">📊 데이터</div> <div class="crawler-tab ${currentTab() === 'settings' ? 'active' : ''}" data-tab="settings" title="설정 변경">⚙️ 설정</div> <div class="crawler-tab ${currentTab() === 'previous-crawls' ? 'active' : ''}" data-tab="previous-crawls" title="이전 크롤링 기록">📁 기록</div> </div> <button id="crawler-minimize" title="창 최소화">─</button> </div> <div id="crawler-content"> <div id="tab-logs" style="display: ${currentTab() === 'logs' ? 'block' : 'none'};"> <div class="progress-bar"> <div class="progress-bar-fill" style="width: ${progress.toFixed(2)}%;"></div> </div> <div class="status-text">진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)</div> <div class="control-buttons"> <button id="start-crawling" title="크롤링을 시작합니다." ${isCrawling ? 'disabled' : ''}>▶ 시작</button> <button id="stop-crawling" title="크롤링을 중지합니다." ${isCrawling ? '' : 'disabled'}>■ 중지</button> ${settings.exportFormats.csv ? `<button id="download-csv" title="CSV 다운로드">📥 CSV</button>` : ''} ${settings.exportFormats.json ? `<button id="download-json" title="JSON 다운로드">📥 JSON</button>` : ''} ${settings.exportFormats.xml ? `<button id="download-xml" title="XML 다운로드">📥 XML</button>` : ''} ${settings.exportFormats.xls ? `<button id="download-xls" title="XLS 다운로드">📥 XLS</button>` : ''} <button id="reset-data" title="모든 크롤링 데이터를 초기화합니다.">🗑️ 초기화</button> </div> <div id="log-messages" style="height: 200px; overflow-y: auto; background: #f9f9f9; padding: 10px; border: 1px solid #ccc; border-radius: 4px;"> 로그 내용이 여기에 표시됩니다. </div> <div id="error-message" class="error-message" style="display: none;"></div> <div id="success-message" class="success-message" style="display: none;"></div> </div> <div id="tab-data" style="display: ${currentTab() === 'data' ? 'block' : 'none'};"> <input type="text" id="search-filter" placeholder="검색 필터..." style="width: 100%; padding: 5px; margin-bottom: 10px; border: 1px solid #ccc; border-radius: 4px;"> <table> <thead> <tr> ${settings.dataFields.name ? '<th data-sort="name">사이트 이름</th>' : ''} ${settings.dataFields.url ? '<th data-sort="url">주소</th>' : ''} ${settings.dataFields.description ? '<th data-sort="description">설명</th>' : ''} ${settings.dataFields.html ? '<th>📄 HTML 내용</th>' : ''} ${settings.dataFields.category ? '<th data-sort="category">분류</th>' : ''} ${settings.dataFields.quality ? '<th data-sort="quality">품질점수</th>' : ''} ${settings.dataFields.language ? '<th data-sort="language">언어</th>' : ''} ${settings.dataFields.wordCount ? '<th data-sort="wordCount">단어 수</th>' : ''} ${settings.dataFields.metaTags ? '<th>메타 태그</th>' : ''} ${settings.dataFields.headings ? '<th>헤딩</th>' : ''} </tr> </thead> <tbody id="data-table-body"></tbody> </table> ${settings.dataVisualization ? ` <div id="data-visualization"> <h3>데이터 시각화</h3> <canvas id="data-chart"></canvas> </div> ` : ''} </div> <div id="tab-settings" style="display: ${currentTab() === 'settings' ? 'block' : 'none'};"> <h3>설정</h3> <div class="settings-section"> <label for="delay-input">크롤링 딜레이 (ms): <span class="tooltip">ℹ️ <span class="tooltiptext">기본 딜레이과 딜레이 랜덤 값을 조합하여 요청 사이의 딜레이를 설정합니다.</span> </span> </label> <input type="number" id="delay-input" value="${settings.delay}" min="500" step="500"> </div> <div class="settings-section"> <label for="delay-random-input">딜레이 랜덤 범위 (ms): <span class="tooltip">ℹ️ <span class="tooltiptext">딜레이에 무작위 요소를 추가하여 보다 자연스러운 요청을 합니다.</span> </span> </label> <input type="number" id="delay-random-input" value="${settings.delayRandom}" min="0" step="500"> </div> <div class="settings-section"> <label for="max-pages-input">최대 크롤링 페이지 수: <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링할 최대 페이지 수를 설정하세요.</span> </span> </label> <input type="number" id="max-pages-input" value="${settings.maxPages}" min="1"> </div> <div class="settings-section"> <label for="user-agent-select">사용자 에이전트(User-Agent): <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링에 사용할 사용자 에이전트를 선택하거나 추가할 수 있습니다.</span> </span> </label> <select id="user-agent-select" multiple size="6"> ${settings.userAgents.map((ua, idx) => `<option value="${idx}" ${idx === settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`).join('')} </select> <button id="add-user-agent" style="margin-top: 5px; padding: 5px 10px; background-color: #2196F3; color: white; border-radius: 4px; font-size: 12px;">추가</button> <button id="remove-user-agent" style="margin-top: 5px; padding: 5px 10px; background-color: #f44336; color: white; border-radius: 4px; font-size: 12px;">제거</button> </div> <div class="settings-section"> <label>콘텐츠 유형:</label> <div> <strong>포함할 유형:</strong> <textarea id="content-include-input" rows="3">${settings.contentTypes.include.join(', ')}</textarea> </div> <div> <strong>제외할 유형:</strong> <textarea id="content-exclude-input" rows="3">${settings.contentTypes.exclude.join(', ')}</textarea> </div> </div> <div class="settings-section"> <label for="timeout-input">크롤링 타임아웃 (ms): <span class="tooltip">ℹ️ <span class="tooltiptext">요청이 완료되기까지 기다릴 최대 시간을 밀리초 단위로 설정합니다.</span> </span> </label> <input type="number" id="timeout-input" value="${settings.timeout}" min="1000" step="1000"> </div> <div class="settings-section"> <label for="include-keywords-input">포함할 키워드: <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링할 페이지에 반드시 포함되어야 하는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="include-keywords-input" rows="3">${settings.includeKeywords.join('\n')}</textarea> </div> <div class="settings-section"> <label for="exclude-keywords-input">제외할 키워드: <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링할 페이지에 포함되면 안 되는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="exclude-keywords-input" rows="3">${settings.excludeKeywords.join('\n')}</textarea> </div> <div class="settings-section"> <label for="regex-include-input">포함할 키워드 (정규 표현식): <span class="tooltip">ℹ️ <span class="tooltiptext">포함할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="regex-include-input" rows="3">${settings.regexInclude.join('\n')}</textarea> </div> <div class="settings-section"> <label for="regex-exclude-input">제외할 키워드 (정규 표현식): <span class="tooltip">ℹ️ <span class="tooltiptext">제외할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="regex-exclude-input" rows="3">${settings.regexExclude.join('\n')}</textarea> </div> <div class="settings-section"> <label for="language-select">언어 필터링:</label> <select id="language-select" multiple size="10"> ${[ { value: 'en', text: '영어 (en)' }, { value: 'ko', text: '한국어 (ko)' }, { value: 'zh', text: '중국어 (zh)' }, { value: 'ja', text: '일본어 (ja)' }, { value: 'ru', text: '러시아어 (ru)' }, { value: 'es', text: '스페인어 (es)' }, { value: 'fr', text: '프랑스어 (fr)' }, { value: 'de', text: '독일어 (de)' }, { value: 'it', text: '이탈리아어 (it)' }, { value: 'pt', text: '포르투갈어 (pt)' }, { value: 'ar', text: '아랍어 (ar)' }, { value: 'hi', text: '힌디어 (hi)' }, { value: 'bn', text: '벵골어 (bn)' }, { value: 'pa', text: '펀자브어 (pa)' }, { value: 'nl', text: '네덜란드어 (nl)' }, { value: 'sv', text: '스웨덴어 (sv)' }, { value: 'tr', text: '터키어 (tr)' }, { value: 'vi', text: '베트남어 (vi)' }, { value: 'pl', text: '폴란드어 (pl)' }, { value: 'th', text: '태국어 (th)' } ].map(lang => `<option value="${lang.value}" ${settings.language.includes(lang.value) ? 'selected' : ''}>${lang.text}</option>`).join('')} </select> </div> <div class="settings-section"> <label for="min-word-count-input">최소 단어 수: <span class="tooltip">ℹ️ <span class="tooltiptext">수집할 페이지의 최소 단어 수를 설정하세요.</span> </span> </label> <input type="number" id="min-word-count-input" value="${settings.minWordCount}" min="10" step="10"> <label for="max-word-count-input">최대 단어 수: <span class="tooltip">ℹ️ <span class="tooltiptext">수집할 페이지의 최대 단어 수를 설정하세요.</span> </span> </label> <input type="number" id="max-word-count-input" value="${settings.maxWordCount}" min="100" step="100"> </div> <div class="settings-section"> <label for="max-depth-input">최대 크롤링 깊이: <span class="tooltip">ℹ️ <span class="tooltiptext">링크를 추적할 최대 깊이를 설정하세요.</span> </span> </label> <input type="number" id="max-depth-input" value="${settings.maxDepth}" min="1" step="1"> </div> <div class="settings-section"> <label for="proxy-input">프록시 서버: <span class="tooltip">ℹ️ <span class="tooltiptext">프록시 서버 주소를 입력하세요. 예: http://proxyserver:port</span> </span> </label> <input type="text" id="proxy-input" value="${settings.proxy}" placeholder="http://proxyserver:port"> </div> <div class="settings-section"> <label>수집할 데이터 필드:</label> <div class="checkbox-group"> ${Object.keys(settings.dataFields).map(field => `<label><input type="checkbox" id="field-${field}" ${settings.dataFields[field] ? 'checked' : ''}> ${getFieldName(field)}</label>`).join('')} </div> </div> <div class="settings-section"> <label for="retry-attempts-input">재시도 횟수: <span class="tooltip">ℹ️ <span class="tooltiptext">요청 실패 시 재시도할 횟수를 설정하세요.</span> </span> </label> <input type="number" id="retry-attempts-input" value="${settings.retryAttempts}" min="0" step="1"> <label for="retry-delay-input">재시도 딜레이 (ms): <span class="tooltip">ℹ️ <span class="tooltiptext">재시도 시 대기할 시간을 밀리초 단위로 설정하세요.</span> </span> </label> <input type="number" id="retry-delay-input" value="${settings.retryDelay}" min="1000" step="500"> </div> <div class="settings-section"> <label>크롤링 스케줄링:</label> <div class="checkbox-group"> <label><input type="checkbox" id="schedule-enabled" ${settings.schedule.enabled ? 'checked' : ''}> 스케줄링 활성화</label> </div> <div id="schedule-options" style="display: ${settings.schedule.enabled ? 'block' : 'none'}; margin-top: 10px;"> <label for="start-time-input">시작 시간 (HH:MM): <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링을 시작할 시간을 설정하세요.</span> </span> </label> <input type="time" id="start-time-input" value="${settings.schedule.startTime}"> <label for="interval-input">크롤링 간격 (분): <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링 작업 간의 간격을 분 단위로 설정하세요.</span> </span> </label> <input type="number" id="interval-input" value="${settings.schedule.interval / 60000}" min="1" step="1"> </div> </div> <div class="settings-section"> <label>크롤링할 검색 엔진 선택:</label> <div class="checkbox-group"> ${Object.keys(settings.searchEngines).map(engine => `<label><input type="checkbox" id="engine-${engine}" ${settings.searchEngines[engine] ? 'checked' : ''}> ${capitalize(engine)}</label>`).join('')} </div> </div> <div class="settings-section"> <label for="blacklist-input">블랙리스트 도메인: <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링에서 제외할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="blacklist-input" rows="5">${settings.blacklist.join('\n')}</textarea> </div> <div class="settings-section"> <label for="whitelist-input">화이트리스트 도메인: <span class="tooltip">ℹ️ <span class="tooltiptext">크롤링을 허용할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.</span> </span> </label> <textarea id="whitelist-input" rows="5">${settings.whitelist.join('\n')}</textarea> </div> <div class="settings-section"> <label for="respect-robots-checkbox">robots.txt 준수: <span class="tooltip">ℹ️ <span class="tooltiptext">대상 사이트의 robots.txt 규칙을 준수하여 크롤링을 진행합니다.</span> </span> </label> <input type="checkbox" id="respect-robots-checkbox" ${settings.respectRobots ? 'checked' : ''}> </div> <div class="settings-section"> <label for="max-concurrent-requests-input">동시 요청 최대 수: <span class="tooltip">ℹ️ <span class="tooltiptext">한 번에 요청할 수 있는 최대 동시 요청 수를 설정하세요.</span> </span> </label> <input type="number" id="max-concurrent-requests-input" value="${settings.maxConcurrentRequests}" min="1" step="1"> </div> <div class="settings-section"> <label for="data-visualization-checkbox">데이터 시각화 활성화: <span class="tooltip">ℹ️ <span class="tooltiptext">수집된 데이터를 시각화하여 그래프로 표시합니다.</span> </span> </label> <input type="checkbox" id="data-visualization-checkbox" ${settings.dataVisualization ? 'checked' : ''}> </div> <div class="settings-section"> <label for="enable-captchas-checkbox">CAPTCHA 처리 활성화: <span class="tooltip">ℹ️ <span class="tooltiptext">CAPTCHA가 감지되었을 때 자동 처리를 활성화합니다. (제한적)</span> </span> </label> <input type="checkbox" id="enable-captchas-checkbox" ${settings.enableCaptchas ? 'checked' : ''}> </div> <div class="settings-section"> <label>데이터 내보내기 형식:</label> <div class="checkbox-group"> ${Object.keys(settings.exportFormats).map(format => `<label><input type="checkbox" id="export-${format}" ${settings.exportFormats[format] ? 'checked' : ''}> ${format.toUpperCase()}</label>`).join('')} </div> </div> <div class="settings-section"> <label for="advanced-logging-checkbox">고급 로깅 활성화: <span class="tooltip">ℹ️ <span class="tooltiptext">추가적인 로깅 정보를 활성화합니다.</span> </span> </label> <input type="checkbox" id="advanced-logging-checkbox" ${settings.advancedLogging ? 'checked' : ''}> </div> <div class="settings-section"> <label for="custom-keywords-input">사용자 지정 키워드 분류: <span class="tooltip">ℹ️ <span class="tooltiptext">사용자 지정 키워드를 추가하여 데이터 분류를 향상시킵니다. 형식: 카테고리:키워드1,키워드2,...</span> </span> </label> <textarea id="custom-keywords-input" rows="5">${formatCustomKeywords(settings.customKeywords)}</textarea> </div> <button id="save-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">저장</button> <button id="reset-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">초기화</button> </div> <div id="tab-previous-crawls" style="display: ${currentTab() === 'previous-crawls' ? 'block' : 'none'};"> <h3>이전 크롤링 기록</h3> <button id="clear-previous-crawls" style="padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px; margin-bottom: 10px;">기록 초기화</button> <table> <thead> <tr> <th>날짜</th> <th>크롤링 수</th> <th>상태</th> </tr> </thead> <tbody id="previous-crawls-body"> ${previousCrawls.map(crawl => ` <tr> <td>${crawl.date}</td> <td>${crawl.count}</td> <td>${crawl.status}</td> </tr> `).join('')} </tbody> </table> </div> </div> <div id="html-modal" class="modal"> <div class="modal-content"> <span class="close">×</span> <pre id="modal-html-content"></pre> </div> </div> </div> `); $('body').append(container); $('#crawler-minimize').on('click', function() { $('#crawler-content').toggle(); const isHidden = $('#crawler-content').is(':hidden'); $(this).text(isHidden ? '☐' : '─'); log(`크롤러 창이 ${isHidden ? '최소화' : '확대'}되었습니다.`, 'info'); }); $('.crawler-tab').on('click', function() { const tab = $(this).data('tab'); $('.crawler-tab').removeClass('active'); $(this).addClass('active'); $('#crawler-content > div').hide(); $(`#tab-${tab}`).show(); saveCurrentTab(tab); }); $(document).on('keydown', function(e) { if (e.ctrlKey) { const actions = { '1': 'logs', '2': 'data', '3': 'settings', '4': 'previous-crawls', 's': 'save', 'r': 'reset', 'p': 'start', 'q': 'stop', 'c': 'csv', 'j': 'json', 'x': 'xml', 'l': 'xls' }; const action = actions[e.key]; if (action) { e.preventDefault(); switch(action) { case 'logs': $('.crawler-tab[data-tab="logs"]').click(); break; case 'data': $('.crawler-tab[data-tab="data"]').click(); break; case 'settings': $('.crawler-tab[data-tab="settings"]').click(); break; case 'previous-crawls': $('.crawler-tab[data-tab="previous-crawls"]').click(); break; case 'save': saveSettings(); break; case 'reset': resetSettings(); break; case 'start': if (!isCrawling) startCrawling(); break; case 'stop': if (isCrawling) stopCrawling(); break; case 'csv': settings.exportFormats.csv && downloadCSV(); break; case 'json': settings.exportFormats.json && downloadJSON(); break; case 'xml': settings.exportFormats.xml && downloadXML(); break; case 'xls': settings.exportFormats.xls && downloadXLS(); break; } } } }); $('#save-settings').on('click', saveSettings); $('#reset-settings').on('click', resetSettings); $('#add-user-agent').on('click', function() { const ua = prompt("새 사용자 에이전트를 입력하세요:"); if (ua) { settings.userAgents.push(ua); GM_setValue('userAgents', settings.userAgents); updateUserAgentSelect(); log('새 사용자 에이전트가 추가되었습니다.', 'success'); } }); $('#remove-user-agent').on('click', function() { const selected = $('#user-agent-select').val(); if (selected && selected.length) { selected.sort((a, b) => b - a).forEach(idx => { settings.userAgents.splice(idx, 1); }); GM_setValue('userAgents', settings.userAgents); settings.userAgentIndex = 0; GM_setValue('userAgentIndex', settings.userAgentIndex); updateUserAgentSelect(); log('선택한 사용자 에이전트가 제거되었습니다.', 'success'); } else alert('제거할 사용자 에이전트를 선택하세요.'); }); $('#download-csv').on('click', downloadCSV); $('#download-json').on('click', downloadJSON); $('#download-xml').on('click', downloadXML); $('#download-xls').on('click', downloadXLS); $('#search-filter').on('input', function() { filterTable($(this).val()); }); $('#schedule-enabled').on('change', function() { $('#schedule-options').toggle($(this).is(':checked')); }); $('#data-visualization-checkbox').on('change', function() { settings.dataVisualization = $(this).is(':checked'); GM_setValue('dataVisualization', settings.dataVisualization); $('#data-visualization').toggle(settings.dataVisualization); log('데이터 시각화 설정이 변경되었습니다.', 'success'); if (settings.dataVisualization) renderChart(); }); $('#enable-captchas-checkbox').on('change', function() { settings.enableCaptchas = $(this).is(':checked'); GM_setValue('enableCaptchas', settings.enableCaptchas); log('CAPTCHA 처리 설정이 변경되었습니다.', 'success'); }); $('.view-html').on('click', function() { const html = decodeURIComponent($(this).data('html')); $('#modal-html-content').text(html); $('#html-modal').show(); }); $('.close').on('click', function() { $('#html-modal').hide(); }); $(window).on('click', function(e) { if (e.target.id === 'html-modal') $('#html-modal').hide(); }); $('#start-crawling').on('click', startCrawling); $('#stop-crawling').on('click', stopCrawling); $('#clear-previous-crawls').on('click', function() { if (confirm("이전 크롤링 기록을 모두 초기화하시겠습니까?")) { previousCrawls = []; GM_setValue('previousCrawls', previousCrawls); $('#previous-crawls-body').empty(); log('이전 크롤링 기록이 초기화되었습니다.', 'success'); } }); $('#reset-data').on('click', function() { if (confirm("모든 크롤링 데이터를 초기화하시겠습니까?")) { data = []; visitedUrls.clear(); GM_setValue('data', []); GM_setValue('visitedUrls', []); updateDataUI(); log('크롤링 데이터가 초기화되었습니다.', 'success'); } }); updateLogsUI(); updateDataUI(); if (settings.dataVisualization) renderChart(); updatePreviousCrawlsUI(); makeDraggable(); } function currentTab() { return GM_getValue('currentTab', 'logs'); } function saveCurrentTab(tab) { GM_setValue('currentTab', tab); } function updateUserAgentSelect() { $('#user-agent-select').empty(); settings.userAgents.forEach((ua, idx) => { $('#user-agent-select').append(`<option value="${idx}" ${idx === settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`); }); } function getCustomKeywordsText() { const ck = settings.customKeywords || {}; return Object.entries(ck).map(([cat, kws]) => `${cat}:${kws.join(',')}`).join('\n'); } function parseCustomKeywords(text) { const lines = text.split('\n').map(l => l.trim()).filter(l => l); const ck = {}; lines.forEach(line => { const [cat, kws] = line.split(':'); if (cat && kws) ck[cat.trim()] = kws.split(',').map(kw => kw.trim()).filter(kw => kw); }); return ck; } function combineKeywords() { const standard = { '교육': ['학교', '교육', '학생', '교수', '강의'], '기술': ['기술', '소프트웨어', '하드웨어', '프로그래밍', '개발'], '뉴스': ['뉴스', '속보', '기사', '보도'], '엔터테인먼트': ['영화', '음악', '연예', '게임', '드라마'], '스포츠': ['축구', '야구', '농구', '경기', '토너먼트'], '비즈니스': ['비즈니스', '경제', '주식', '투자', '시장'], '건강': ['건강', '의료', '헬스', '운동', '영양'], '과학': ['과학', '연구', '실험', '발견', '이론'], '문학': ['소설', '시', '문학', '작가', '책'], '여행': ['여행', '관광', '숙박', '비행기', '여권'] }; const custom = settings.customKeywords || {}; for (const [cat, kws] of Object.entries(custom)) { standard[cat] = standard[cat] ? [...new Set([...standard[cat], ...kws])] : kws; } return standard; } function log(message, type = 'info') { const timestamp = new Date().toLocaleString(); logs.push({ timestamp, message, type }); GM_setValue('logs', logs); updateLogsUI(); if (settings.advancedLogging) { const styles = { 'info': 'color: blue;', 'warning': 'color: orange;', 'error': 'color: red;', 'success': 'color: green; font-weight: bold;' }; console.log(`%c[${timestamp}] [${type.toUpperCase()}] ${message}`, styles[type] || ''); } } function updateLogsUI() { $('#log-messages').html(logs.map(l => `<div>[${l.timestamp}] [${l.type.toUpperCase()}] ${l.message}</div>`).join('')); $('.progress-bar-fill').css('width', `${progress.toFixed(2)}%`); $('.status-text').text(`진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)`); // Enhance error message visibility const latestLog = logs[logs.length - 1]; if (latestLog && latestLog.type === 'error') { $('#error-message').text(latestLog.message).show(); setTimeout(() => { $('#error-message').fadeOut(); }, 5000); } if (latestLog && latestLog.type === 'success') { $('#success-message').text(latestLog.message).show(); setTimeout(() => { $('#success-message').fadeOut(); }, 5000); } } function updateDataUI() { const filter = $('#search-filter').val().toLowerCase(); const filtered = filter ? data.filter(item => Object.keys(settings.dataFields).some(field => settings.dataFields[field] && String(item[field]).toLowerCase().includes(filter))) : data; const rows = filtered.map(item => ` <tr> ${settings.dataFields.name ? `<td>${escapeHTML(item.name)}</td>` : ''} ${settings.dataFields.url ? `<td><a href="${escapeHTML(item.url)}" target="_blank">${escapeHTML(item.url)}</a></td>` : ''} ${settings.dataFields.description ? `<td>${escapeHTML(item.description)}</td>` : ''} ${settings.dataFields.html ? `<td><button class="view-html" data-html="${encodeURIComponent(item.html)}">📄 보기</button></td>` : ''} ${settings.dataFields.category ? `<td>${escapeHTML(item.category)}</td>` : ''} ${settings.dataFields.quality ? `<td>${escapeHTML(item.quality.toString())}</td>` : ''} ${settings.dataFields.language ? `<td>${escapeHTML(item.language)}</td>` : ''} ${settings.dataFields.wordCount ? `<td>${escapeHTML(item.wordCount.toString())}</td>` : ''} ${settings.dataFields.metaTags ? `<td>${escapeHTML(JSON.stringify(item.metaTags))}</td>` : ''} ${settings.dataFields.headings ? `<td>${escapeHTML(item.headings.join(', '))}</td>` : ''} </tr> `).join(''); $('#data-table-body').html(rows); $('.view-html').off('click').on('click', function() { $('#modal-html-content').text(decodeURIComponent($(this).data('html'))); $('#html-modal').show(); }); if (settings.dataVisualization) renderChart(); } function renderChart() { if (!document.getElementById('data-chart').getContext) return; const ctx = document.getElementById('data-chart').getContext('2d'); if (window.dataChartInstance) window.dataChartInstance.destroy(); const categories = {}; data.forEach(item => { const cat = item.category || '기타'; categories[cat] = (categories[cat] || 0) + 1; }); const chartData = { labels: Object.keys(categories), datasets: [{ label: '카테고리별 데이터 수', data: Object.values(categories), backgroundColor: generateColors(Object.keys(categories).length) }] }; window.dataChartInstance = new Chart(ctx, { type: 'pie', data: chartData, options: { responsive: true } }); } function generateColors(count) { return Array.from({length: count}, (_, i) => `hsl(${i * (360 / count)}, 70%, 50%)`); } function escapeHTML(str) { return String(str).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'"); } function downloadCSV() { if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; } const headers = Object.keys(settings.dataFields).filter(f => settings.dataFields[f]).map(f => getFieldName(f)); const csv = [headers.join(",")].concat(data.map(item => Object.keys(settings.dataFields).filter(f => settings.dataFields[f]).map(f => `"${escapeCSV(item[f] || '')}"`).join(","))).join("\n"); const link = document.createElement("a"); link.href = `data:text/csv;charset=utf-8,${encodeURIComponent(csv)}`; link.download = "data.csv"; document.body.appendChild(link); link.click(); document.body.removeChild(link); log('CSV 파일이 다운로드되었습니다.', 'success'); } function downloadJSON() { if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; } const json = JSON.stringify(data, null, 2); const blob = new Blob([json], { type: "application/json" }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); link.download = "data.json"; document.body.appendChild(link); link.click(); document.body.removeChild(link); log('JSON 파일이 다운로드되었습니다.', 'success'); } function downloadXML() { if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; } let xml = '<?xml version="1.0" encoding="UTF-8"?><data>'; data.forEach(item => { xml += '<entry>'; if (settings.dataFields.name) xml += `<name>${escapeXML(item.name || 'No Title')}</name>`; if (settings.dataFields.url) xml += `<url>${escapeXML(item.url || '')}</url>`; if (settings.dataFields.description) xml += `<description>${escapeXML(item.description || 'No Description')}</description>`; if (settings.dataFields.html) xml += `<html><![CDATA[${item.html || ''}]]></html>`; if (settings.dataFields.category) xml += `<category>${escapeXML(item.category || '')}</category>`; if (settings.dataFields.quality) xml += `<quality>${escapeXML(item.quality)}</quality>`; if (settings.dataFields.language) xml += `<language>${escapeXML(item.language || 'unknown')}</language>`; if (settings.dataFields.wordCount) xml += `<wordCount>${escapeXML(item.wordCount)}</wordCount>`; if (settings.dataFields.metaTags) xml += `<metaTags>${escapeXML(JSON.stringify(item.metaTags))}</metaTags>`; if (settings.dataFields.headings) xml += `<headings>${escapeXML(item.headings.join('; '))}</headings>`; xml += '</entry>'; }); xml += '</data>'; const blob = new Blob([xml], { type: "application/xml" }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); link.download = "data.xml"; document.body.appendChild(link); link.click(); document.body.removeChild(link); log('XML 파일이 다운로드되었습니다.', 'success'); } function downloadXLS() { if (!data.length) { log('다운로드할 데이터가 없습니다.', 'warning'); return; } const sheet = XLSX.utils.json_to_sheet(data.map(item => { const entry = {}; if (settings.dataFields.name) entry['사이트 이름'] = item.name; if (settings.dataFields.url) entry['주소'] = item.url; if (settings.dataFields.description) entry['설명'] = item.description; if (settings.dataFields.html) entry['📄 HTML 내용'] = item.html; if (settings.dataFields.category) entry['분류'] = item.category; if (settings.dataFields.quality) entry['품질점수'] = item.quality; if (settings.dataFields.language) entry['언어'] = item.language; if (settings.dataFields.wordCount) entry['단어 수'] = item.wordCount; if (settings.dataFields.metaTags) entry['메타 태그'] = JSON.stringify(item.metaTags); if (settings.dataFields.headings) entry['헤딩'] = item.headings.join('; '); return entry; })); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, sheet, "Data"); XLSX.writeFile(wb, "data.xlsx"); log('XLS 파일이 다운로드되었습니다.', 'success'); } function escapeCSV(str) { return String(str).replace(/"/g, '""'); } function escapeXML(str) { return String(str).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'"); } function filterTable(keyword) { $('#data-table-body tr').each(function() { $(this).toggle($(this).text().toLowerCase().includes(keyword.toLowerCase())); }); } function saveSettings() { // Input Validation const delay = parseInt($('#delay-input').val(), 10); const delayRandom = parseInt($('#delay-random-input').val(), 10); const maxPages = parseInt($('#max-pages-input').val(), 10); const timeout = parseInt($('#timeout-input').val(), 10); const minWordCount = parseInt($('#min-word-count-input').val(), 10); const maxWordCount = parseInt($('#max-word-count-input').val(), 10); const maxDepth = parseInt($('#max-depth-input').val(), 10); const retryAttempts = parseInt($('#retry-attempts-input').val(), 10); const retryDelay = parseInt($('#retry-delay-input').val(), 10); const interval = parseInt($('#interval-input').val(), 10) * 60000; const isValidSettings = !isNaN(delay) && delay >= 500 && !isNaN(delayRandom) && delayRandom >= 0 && !isNaN(maxPages) && maxPages >= 1 && !isNaN(timeout) && timeout >= 1000 && !isNaN(minWordCount) && minWordCount >= 10 && !isNaN(maxWordCount) && maxWordCount >= 100 && !isNaN(maxDepth) && maxDepth >= 1 && !isNaN(retryAttempts) && retryAttempts >= 0 && !isNaN(retryDelay) && retryDelay >= 1000 && (!settings.schedule.enabled || (!isNaN(interval) && interval >= 60000)); if (!isValidSettings) { $('#error-message').text('설정값이 유효하지 않습니다. 올바른 값을 입력하세요.').show(); setTimeout(() => { $('#error-message').fadeOut(); }, 5000); return; } settings.delay = delay; settings.delayRandom = delayRandom; settings.maxPages = maxPages; const selectedUA = parseInt($('#user-agent-select').val(), 10); settings.userAgentIndex = isNaN(selectedUA) ? 0 : selectedUA; settings.contentTypes.include = $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(ct => ct) || defaultSettings.contentTypes.include; settings.contentTypes.exclude = $('#content-exclude-input').val().split(',').map(ct => ct.trim()).filter(ct => ct) || defaultSettings.contentTypes.exclude; settings.timeout = timeout; settings.includeKeywords = $('#include-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw) || defaultSettings.includeKeywords; settings.excludeKeywords = $('#exclude-keywords-input').val().split('\n').map(kw => kw.trim()).filter(kw => kw) || defaultSettings.excludeKeywords; settings.regexInclude = $('#regex-include-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx) || defaultSettings.regexInclude; settings.regexExclude = $('#regex-exclude-input').val().split('\n').map(rx => rx.trim()).filter(rx => rx) || defaultSettings.regexExclude; settings.language = $('#language-select').val() || []; settings.minWordCount = minWordCount; settings.maxWordCount = maxWordCount; settings.maxDepth = maxDepth; settings.proxy = $('#proxy-input').val().trim() || defaultSettings.proxy; ['name','url','description','html','category','quality','language','wordCount','metaTags','headings'].forEach(field => { settings.dataFields[field] = $(`#field-${field}`).is(':checked'); }); settings.retryAttempts = retryAttempts; settings.retryDelay = retryDelay; settings.schedule.enabled = $('#schedule-enabled').is(':checked'); settings.schedule.startTime = $('#start-time-input').val() || defaultSettings.schedule.startTime; settings.schedule.interval = settings.schedule.enabled ? (isNaN(interval) ? defaultSettings.schedule.interval : interval) : defaultSettings.schedule.interval; Object.keys(settings.searchEngines).forEach(engine => { settings.searchEngines[engine] = $(`#engine-${engine}`).is(':checked'); }); settings.blacklist = $('#blacklist-input').val().split('\n').map(d => d.trim()).filter(d => d) || defaultSettings.blacklist; settings.whitelist = $('#whitelist-input').val().split('\n').map(d => d.trim()).filter(d => d) || defaultSettings.whitelist; settings.respectRobots = $('#respect-robots-checkbox').is(':checked'); settings.maxConcurrentRequests = parseInt($('#max-concurrent-requests-input').val(), 10) || defaultSettings.maxConcurrentRequests; settings.dataVisualization = $('#data-visualization-checkbox').is(':checked'); settings.exportFormats.csv = $('#export-csv').is(':checked'); settings.exportFormats.json = $('#export-json').is(':checked'); settings.exportFormats.xml = $('#export-xml').is(':checked'); settings.exportFormats.xls = $('#export-xls').is(':checked'); settings.advancedLogging = $('#advanced-logging-checkbox').is(':checked'); settings.enableCaptchas = $('#enable-captchas-checkbox').is(':checked'); const customKeywords = parseCustomKeywords($('#custom-keywords-input').val()); settings.customKeywords = customKeywords; // Save Settings GM_setValue('delay', settings.delay); GM_setValue('delayRandom', settings.delayRandom); GM_setValue('maxPages', settings.maxPages); GM_setValue('userAgentIndex', settings.userAgentIndex); GM_setValue('contentTypes', settings.contentTypes); GM_setValue('timeout', settings.timeout); GM_setValue('includeKeywords', settings.includeKeywords); GM_setValue('excludeKeywords', settings.excludeKeywords); GM_setValue('regexInclude', settings.regexInclude); GM_setValue('regexExclude', settings.regexExclude); GM_setValue('language', settings.language); GM_setValue('minWordCount', settings.minWordCount); GM_setValue('maxWordCount', settings.maxWordCount); GM_setValue('maxDepth', settings.maxDepth); GM_setValue('proxy', settings.proxy); GM_setValue('dataFields', settings.dataFields); GM_setValue('retryAttempts', settings.retryAttempts); GM_setValue('retryDelay', settings.retryDelay); GM_setValue('schedule', settings.schedule); GM_setValue('searchEngines', settings.searchEngines); GM_setValue('blacklist', settings.blacklist); GM_setValue('whitelist', settings.whitelist); GM_setValue('respectRobots', settings.respectRobots); GM_setValue('maxConcurrentRequests', settings.maxConcurrentRequests); GM_setValue('dataVisualization', settings.dataVisualization); GM_setValue('exportFormats', settings.exportFormats); GM_setValue('advancedLogging', settings.advancedLogging); GM_setValue('enableCaptchas', settings.enableCaptchas); GM_setValue('customKeywords', settings.customKeywords); totalPages = settings.maxPages; GM_setValue('totalPages', totalPages); log('설정이 저장되었습니다.', 'success'); if (settings.schedule.enabled) { scheduleCrawling(); } updateDataUI(); if (settings.dataVisualization) renderChart(); } function resetSettings() { if (confirm("기본 설정으로 초기화하시겠습니까?")) { Object.keys(defaultSettings).forEach(key => GM_deleteValue(key)); Object.assign(settings, defaultSettings); ['userAgents','userAgentIndex','contentTypes','dataFields','exportFormats','advancedLogging','enableCaptchas'].forEach(key => GM_setValue(key, settings[key])); GM_setValue('customKeywords', {}); GM_setValue('logs', []); GM_setValue('data', []); GM_setValue('visitedUrls', []); GM_setValue('previousCrawls', []); $('#custom-keywords-input').val(''); updateUserAgentSelect(); updateDataUI(); updateLogsUI(); $('#previous-crawls-body').empty(); if (settings.dataVisualization) renderChart(); log('기본 설정으로 초기화되었습니다.', 'success'); } } async function startCrawling() { if (isCrawling) { log('이미 크롤링이 진행 중입니다.', 'warning'); return; } if (!Object.values(settings.searchEngines).some(v => v)) { log('사용할 검색 엔진이 하나도 선택되지 않았습니다.', 'error'); return; } isCrawling = true; GM_setValue('isCrawling', isCrawling); log('크롤링을 시작합니다.', 'info'); setButtonStates(); currentPage = parseInt(GM_getValue('currentPage', 1), 10); totalPages = parseInt(GM_getValue('totalPages', settings.maxPages), 10); GM_setValue('totalPages', totalPages); currentDepth = parseInt(GM_getValue('currentDepth', 0), 10); if (!previousCrawls.some(crawl => crawl.status === '진행 중')) { previousCrawls.push({ date: new Date().toLocaleString(), count: 0, status: '진행 중' }); GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } await crawlPage(currentDepth); } function stopCrawling() { if (!isCrawling) { log('크롤링이 진행 중이지 않습니다.', 'warning'); return; } isCrawling = false; GM_setValue('isCrawling', isCrawling); log('크롤링이 중지되었습니다.', 'warning'); finalizeCrawl('중지됨'); setButtonStates(); } function setButtonStates() { $('#start-crawling').prop('disabled', isCrawling); $('#stop-crawling').prop('disabled', !isCrawling); } async function crawlPage(depth) { try { while (isCrawling && currentPage <= totalPages && depth <= settings.maxDepth) { log(`페이지 ${currentPage} 크롤링 시작. (깊이: ${depth})`, 'info'); if (settings.respectRobots && !(await checkRobotsTxt(window.location.href))) { log(`robots.txt에 의해 크롤링이 금지된 페이지: ${window.location.href}`, 'warning'); break; } const searchEngine = identifySearchEngine(); if (!searchEngine || !searchEngineHandlers[searchEngine]) { log(`지원되지 않는 검색 엔진: ${window.location.hostname}`, 'error'); break; } const doc = document; const handler = searchEngineHandlers[searchEngine]; const results = handler.extractResults(doc); if (!results) { log('검색 결과를 찾을 수 없습니다.', 'error'); break; } results.forEach(result => { if (!isCrawling) return; const { name, url, description } = result; if (visitedUrls.has(url) || !isValidURL(url) || (settings.whitelist.length && !settings.whitelist.some(d => url.includes(d))) || settings.blacklist.some(d => url.includes(d)) || (settings.includeKeywords.length && !settings.includeKeywords.some(k => url.includes(k) || description.includes(k))) || (settings.excludeKeywords.some(k => url.includes(k) || description.includes(k))) || (settings.regexInclude.length && !settings.regexInclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description))) || (settings.regexExclude.length && settings.regexExclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(description)))) { return; } requestQueue.push({ url, depth }); if (activeRequests >= settings.maxConcurrentRequests) return; }); if (previousCrawls.length && previousCrawls[previousCrawls.length -1].status === '진행 중') { previousCrawls[previousCrawls.length -1].count += results.length; GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } await processQueue(); progress = (currentPage / totalPages) * 100; GM_setValue('progress', progress); updateLogsUI(); if (currentPage < totalPages && isCrawling) { const nextUrl = handler.getNextPageUrl(doc); if (nextUrl) { currentPage++; GM_setValue('currentPage', currentPage); depth++; GM_setValue('currentDepth', depth); log(`다음 페이지로 이동: ${currentPage}`, 'info'); await dynamicDelay(); const nextPageContent = await fetchPageContent(nextUrl); if (nextPageContent) { updatePageContent(nextPageContent); continue; } else { log('다음 페이지 내용을 가져올 수 없습니다.', 'error'); break; } } else { log('다음 페이지 링크를 찾을 수 없습니다.', 'error'); break; } } else { break; } } if (isCrawling && currentPage > totalPages && activeRequests === 0 && requestQueue.length === 0) { log('크롤링이 완료되었습니다.', 'success'); finalizeCrawl('완료됨'); isCrawling = false; GM_setValue('isCrawling', isCrawling); setButtonStates(); GM_setValue('progress', 100); $('.progress-bar-fill').css('width', `100%`); $('.status-text').text(`진행률: 100% (${currentPage}/${totalPages} 페이지)`); } } catch (error) { log(`크롤링 중 오류 발생: ${error.message}`, 'error'); finalizeCrawl('오류 발생'); } } async function processQueue() { while (requestQueue.length && activeRequests < settings.maxConcurrentRequests && isCrawling) { const { url, depth } = requestQueue.shift(); activeRequests++; handleRequest(url, depth).finally(() => { activeRequests--; processQueue(); }); } } async function handleRequest(url, depth) { if (!isCrawling || visitedUrls.has(url)) return; try { const response = await fetchWithRetry(url, settings.retryAttempts, settings.retryDelay); const contentType = getContentType(response); if (!isContentTypeAllowed(contentType)) { log(`허용되지 않은 콘텐츠 유형(${contentType})의 URL: ${url}`, 'warning'); return; } const parser = new DOMParser(); const doc = parser.parseFromString(response.responseText, 'text/html'); const text = doc.body ? doc.body.innerText : ''; if (settings.enableCaptchas && detectCaptcha(doc)) { await handleCaptcha(url); return; } const wc = text.split(/\s+/).length; if (wc < settings.minWordCount || wc > settings.maxWordCount) { log(`단어 수 조건에 맞지 않는 URL: ${url} (${wc} 단어)`, 'warning'); return; } const quality = evaluateContentQuality(text); const lang = doc.documentElement.lang || 'unknown'; const category = classifyKeywords(text); const meta = extractMetaTags(doc); const headings = extractHeadings(doc); if (category !== '기타' || quality > 100) { const entry = { url, quality, language: lang, wordCount: wc, category }; if (settings.dataFields.name) entry.name = extractName(doc) || 'No Title'; if (settings.dataFields.description) entry.description = extractDescription(doc) || 'No Description'; if (settings.dataFields.html) entry.html = response.responseText; if (settings.dataFields.metaTags) entry.metaTags = meta; if (settings.dataFields.headings) entry.headings = headings; data.push(entry); GM_setValue('data', JSON.stringify([...data])); updateDataUI(); log(`데이터 수집 완료: ${url}`, 'success'); if (settings.dataVisualization) renderChart(); } if (depth < settings.maxDepth) { const links = extractLinks(doc); links.forEach(link => { if (!visitedUrls.has(link) && isValidURL(link)) requestQueue.push({ url: link, depth: depth + 1 }); }); } } catch (error) { log(`데이터 수집 실패: ${url} - ${error.message}`, 'error'); } visitedUrls.add(url); GM_setValue('visitedUrls', JSON.stringify([...visitedUrls])); } async function fetchWithRetry(url, attempts, delayMs) { for (let i = 1; i <= attempts; i++) { try { return await GM_xmlhttpRequestPromise(url); } catch (e) { log(`요청 실패 (${i}/${attempts}): ${url} - ${e.message}`, 'warning'); if (i < attempts) await delay(delayMs); } } throw new Error('모든 재시도 시도가 실패했습니다.'); } function isAnySearchEngineEnabled() { return Object.values(settings.searchEngines).some(v => v); } function classifyKeywords(text) { const kws = combineKeywords(); for (const [cat, keywords] of Object.entries(kws)) { if (keywords.some(kw => text.includes(kw))) return cat; } return '기타'; } function extractSearchResults() { // Deprecated: Using handlers now } function getNextPageUrl() { // Deprecated: Using handlers now } function identifySearchEngine() { const host = window.location.hostname; if (host.includes('google.com')) return 'google'; if (host.includes('bing.com')) return 'bing'; if (host.includes('yahoo.com')) return 'yahoo'; if (host.includes('duckduckgo.com')) return 'duckduckgo'; // Add other search engines as needed return null; } function GM_xmlhttpRequestPromise(url) { return new Promise((resolve, reject) => { let reqUrl = settings.proxy ? `${settings.proxy}?url=${encodeURIComponent(url)}` : url; GM_xmlhttpRequest({ method: "GET", url: reqUrl, headers: { 'User-Agent': getCurrentUserAgent(), 'Accept': '*/*' }, timeout: settings.timeout, onload: res => (res.status >= 200 && res.status < 300) ? resolve(res) : reject(new Error(`HTTP ${res.status}`)), onerror: () => reject(new Error('Request failed')), ontimeout: () => reject(new Error('Request timed out')) }); }); } function getCurrentUserAgent() { return settings.userAgents[settings.userAgentIndex % settings.userAgents.length]; } function getContentType(res) { return res.responseHeaders.split('\n').find(h => h.toLowerCase().startsWith('content-type'))?.split(':')[1].trim().split(';')[0] || ''; } function isContentTypeAllowed(ct) { if (settings.contentTypes.include.length && !settings.contentTypes.include.includes(ct)) return false; if (settings.contentTypes.exclude.includes(ct)) return false; return true; } function evaluateContentQuality(text) { return text.split(/\s+/).length; } function extractMetaTags(doc) { const metas = {}; doc.querySelectorAll('meta[name], meta[property]').forEach(meta => { const name = meta.getAttribute('name') || meta.getAttribute('property'); const content = meta.getAttribute('content'); if (name && content) metas[name.toLowerCase()] = content; }); return metas; } function extractHeadings(doc) { const hs = []; for (let i = 1; i <= 6; i++) doc.querySelectorAll(`h${i}`).forEach(h => hs.push(h.innerText.trim())); return hs; } function isValidURL(url) { try { new URL(url); return true; } catch { return false; } } async function checkRobotsTxt(url) { try { const parsed = new URL(url); const robots = `${parsed.origin}/robots.txt`; const res = await GM_xmlhttpRequestPromise(robots); return new RobotsParser(robots, res.responseText).isAllowed(url, getCurrentUserAgent()); } catch { log('robots.txt 확인 실패, 기본 허용.', 'warning'); return true; } } class RobotsParser { constructor(url, text) { this.rules = []; this.parse(text); } parse(text) { let agents = []; text.split('\n').forEach(line => { const [key, ...val] = line.split(':').map(s => s.trim()); if (!key) return; if (key.toLowerCase() === 'user-agent') { agents = [val.join(':').toLowerCase()]; } else if (['disallow','allow'].includes(key.toLowerCase())) this.rules.push({ agents, type: key.toLowerCase(), path: val.join(':') }); }); } isAllowed(url, ua) { const path = new URL(url).pathname; let allowed = true; this.rules.forEach(rule => { if (rule.agents.includes('*') || rule.agents.includes(ua.toLowerCase())) { if (rule.type === 'disallow' && path.startsWith(rule.path)) allowed = false; if (rule.type === 'allow' && path.startsWith(rule.path)) allowed = true; } }); return allowed; } } function delay(ms) { return new Promise(res => setTimeout(res, ms)); } function formatCustomKeywords(ck) { return Object.entries(ck).map(([c, kws]) => `${c}:${kws.join(',')}`).join('\n'); } function getFieldName(field) { return { name: '사이트 이름', url: '주소', description: '설명', html: '📄 HTML 내용', category: '분류', quality: '품질점수', language: '언어', wordCount: '단어 수', metaTags: '메타 태그', headings: '헤딩' }[field]; } function capitalize(str) { return str.charAt(0).toUpperCase() + str.slice(1); } async function handleCaptcha(url) { log(`CAPTCHA 감지됨: ${url}`, 'warning'); // CAPTCHA 처리 로직 추가 (제한적) // 예: CAPTCHA 해결을 위한 별도의 서비스 연동 또는 수동 해결 유도 } function extractName(doc) { return doc.querySelector('title')?.innerText || 'No Title'; } function extractDescription(doc) { return doc.querySelector('meta[name="description"]')?.getAttribute('content') || 'No Description'; } function extractLinks(doc) { return Array.from(doc.querySelectorAll('a[href]')).map(a => a.href).filter(isValidURL); } function escapeXML(str) { return String(str).replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'"); } async function scheduleCrawling() { const now = new Date(); const [startHour, startMinute] = settings.schedule.startTime.split(':').map(Number); const startTime = new Date(now.getFullYear(), now.getMonth(), now.getDate(), startHour, startMinute, 0); if (startTime < now) startTime.setDate(startTime.getDate() + 1); const initialDelay = startTime - now; setTimeout(() => { startCrawling(); setInterval(startCrawling, settings.schedule.interval); }, initialDelay); log(`크롤링 스케줄이 설정되었습니다. 시작 시간: ${startTime.toLocaleString()}, 간격: ${settings.schedule.interval / 60000}분`, 'info'); } function finalizeCrawl(status) { if (previousCrawls.length) { previousCrawls[previousCrawls.length -1].status = status; GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } } function updatePreviousCrawlsUI() { $('#previous-crawls-body').html(previousCrawls.map(crawl => ` <tr> <td>${crawl.date}</td> <td>${crawl.count}</td> <td>${crawl.status}</td> </tr> `).join('')); } function init() { createUI(); if (isCrawling) { log('크롤링 재개됨.', 'info'); setButtonStates(); crawlPage(currentDepth); } else log('크롤러 초기화됨.', 'info'); } window.addEventListener('load', init); function makeDraggable() { const container = document.getElementById('crawler-container'); const header = document.getElementById('crawler-header'); let isDragging = false; let startX, startY, initialX, initialY; header.onmousedown = function(e) { isDragging = true; startX = e.clientX; startY = e.clientY; initialX = container.offsetLeft; initialY = container.offsetTop; document.onmousemove = drag; document.onmouseup = stopDrag; return false; }; function drag(e) { if (isDragging) { let dx = e.clientX - startX; let dy = e.clientY - startY; container.style.left = (initialX + dx) + 'px'; container.style.top = (initialY + dy) + 'px'; } } function stopDrag() { isDragging = false; document.onmousemove = null; document.onmouseup = null; } } async function startCrawling() { if (isCrawling) { log('이미 크롤링이 진행 중입니다.', 'warning'); return; } if (!Object.values(settings.searchEngines).some(v => v)) { log('사용할 검색 엔진이 하나도 선택되지 않았습니다.', 'error'); return; } isCrawling = true; GM_setValue('isCrawling', isCrawling); log('크롤링을 시작합니다.', 'info'); setButtonStates(); currentPage = parseInt(GM_getValue('currentPage', 1), 10); totalPages = parseInt(GM_getValue('totalPages', settings.maxPages), 10); GM_setValue('totalPages', totalPages); currentDepth = parseInt(GM_getValue('currentDepth', 0), 10); if (!previousCrawls.some(crawl => crawl.status === '진행 중')) { previousCrawls.push({ date: new Date().toLocaleString(), count: 0, status: '진행 중' }); GM_setValue('previousCrawls', previousCrawls); updatePreviousCrawlsUI(); } await crawlPage(currentDepth); } // Add additional functions or modifications as needed })();