// ==UserScript== // @name Advanced Web Crawler Enhanced Pro Optimized v6.3.4 //…
- User
// ==UserScript== // @name Advanced Web Crawler Enhanced Pro Optimized v6.3.4 // @namespace http://tampermonkey.net/ // @version 6.3.8 // @description Further optimized and enhanced version with responsive design and other improvements. // @author // @match https://www.google.com/search* // @match https://www.bing.com/search* // @match https://search.yahoo.com/search* // @match https://duckduckgo.com/* // @match https://www.baidu.com/s* // @match https://www.yandex.com/search/* // @match https://search.naver.com/search* // @match https://search.daum.net/search* // @match https://www.ask.com/* // @match https://www.ecosia.org/search* // @match https://www.startpage.com/* // @match https://searx.me/* // @match https://www.dogpile.com/* // @match https://www.bing.com/images* // @grant GM_xmlhttpRequest // @grant GM_setValue // @grant GM_getValue // @grant GM_deleteValue // @grant GM_addStyle // @connect * // @require https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js // @require https://cdn.jsdelivr.net/npm/chart.js // @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js // @require https://cdnjs.cloudflare.com/ajax/libs/mocha/10.2.0/mocha.min.js // @require https://cdnjs.cloudflare.com/ajax/libs/chai/4.3.7/chai.min.js // @require https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js // ==/UserScript==
(function() { 'use strict';
const Config = (() => { const DEFAULT_SETTINGS = { delay: 2000, delayRandom: 500, maxPages: 5, userAgents: [ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)', 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15', 'Mozilla/5.0 (iPad; CPU OS 14_0 like Mac OS X) AppleWebKit/605.1.15', 'AdvancedWebCrawler/6.3 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' ], userAgentIndex: 0, blacklist: [], whitelist: [], contentTypes: { include: ['text/html'], exclude: ['application/pdf', 'image/jpeg', 'image/png', 'video/mp4'] }, timeout: 8000, includeKeywords: [], excludeKeywords: [], language: [], minWordCount: 100, maxWordCount: 10000, maxDepth: 2, proxy: '', regexInclude: [], regexExclude: [], dataFields: { name: true, url: true, description: true, html: true, category: true, quality: true, language: true, wordCount: true, metaTags: false, headings: false }, schedule: { enabled: false, startTime: '00:00', interval: 3600000, specificDays: [] }, searchEngines: { google: true, bing: true, yahoo: true, duckduckgo: true, baidu: true, yandex: true, naver: true, daum: true, ask: true, ecosia: true, startpage: true, searx: true, dogpile: true, bingimages: true }, retryAttempts: 3, retryDelay: 2000, respectRobots: true, maxConcurrentRequests: 3, dataVisualization: true, exportFormats: { csv: true, json: true, xml: true, xls: true }, advancedLogging: true, enableCaptchas: false, customKeywords: {}, templates: {}, currentTab: 'logs', isCrawling: false, darkMode: false }; let settings = { ...DEFAULT_SETTINGS, ...getSavedSettings() }; function getSavedSettings() { return GM_getValue('settings', DEFAULT_SETTINGS); } function saveSettings(newSettings) { settings = { ...settings, ...newSettings }; GM_setValue('settings', settings); } function resetSettings() { settings = { ...DEFAULT_SETTINGS }; GM_setValue('settings', settings); } function saveTemplate(name) { settings.templates[name] = { ...settings }; saveSettings({}); Logger.log(`설정 템플릿 '${name}'이(가) 저장되었습니다.`, 'success'); } function loadTemplate(name) { if (name in settings.templates) { settings = { ...settings, ...settings.templates[name] }; saveSettings({}); Logger.log(`설정 템플릿 '${name}'이(가) 불러와졌습니다.`, 'success'); } else { Logger.log(`설정 템플릿 '${name}'을(를) 찾을 수 없습니다.`, 'error'); } } function validateProxy(proxyUrl) { try { const url = new URL(proxyUrl); return ['http:', 'https:'].includes(url.protocol); } catch { return false; } } return { DEFAULT_SETTINGS, get settings() { return settings; }, saveSettings, resetSettings, saveTemplate, loadTemplate, validateProxy }; })(); const Logger = (() => { let logs = JSON.parse(GM_getValue('logs', '[]')); function log(message, type = 'info') { const timestamp = new Date().toLocaleString(); logs.push({ timestamp, message, type }); if (logs.length > 1000) logs.shift(); GM_setValue('logs', JSON.stringify(logs)); UI.updateLogs(); if (Config.settings.advancedLogging) { const styles = { info: 'color: blue;', warning: 'color: orange;', error: 'color: red;', success: 'color: green; font-weight: bold;', debug: 'color: purple;' }; console.log(`%c[${timestamp}] [${type.toUpperCase()}] ${message}`, styles[type] || ''); } } function getLogs() { return logs; } function clearLogs() { logs = []; GM_setValue('logs', JSON.stringify(logs)); UI.updateLogs(); } return { log, getLogs, clearLogs }; })(); const Utils = (() => { const escapeMap = { html: { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }, csv: { '"': '""' }, xml: { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' } }; function escape(str, type = 'html') { return String(str).replace(new RegExp(`[${Object.keys(escapeMap[type]).join('')}]`, 'g'), match => escapeMap[type][match]); } function capitalize(str) { return str.charAt(0).toUpperCase() + str.slice(1); } function delay(ms) { return new Promise(res => setTimeout(res, ms)); } function generateColors(count) { return Array.from({ length: count }, (_, i) => `hsl(${i * (360 / count)}, 70%, 50%)`); } function parseCustomKeywords(text) { const lines = text.split('\n').map(l => l.trim()).filter(l => l); return lines.reduce((acc, line) => { const [cat, kws] = line.split(':').map(s => s.trim()); if (cat && kws) acc[cat] = kws.split(',').map(kw => kw.trim()).filter(Boolean); return acc; }, {}); } function formatCustomKeywords(ck) { return Object.entries(ck).map(([c, kws]) => `${c}:${kws.join(',')}`).join('\n'); } function isValidURL(url) { try { new URL(url); return true; } catch { return false; } } return { escape, capitalize, delay, generateColors, parseCustomKeywords, formatCustomKeywords, isValidURL }; })(); const UI = (() => { function createUI() { GM_addStyle(` /* Highlight.js default style */ .hljs { display: block; overflow-x: auto; padding: 0.5em; background: #f0f0f0; } /* Dark mode adjustments for Highlight.js */ #crawler-container.dark .hljs { background: #2b2b2b; color: #f1f1f1; } /* Existing styles */ #crawler-container { position: fixed; top: 10px; right: 10px; width: 400px; /* Reduced width */ max-width: 500px; /* Reduced max width */ height: 600px; /* Reduced height */ background: var(--bg-color, #fff); border: 1px solid #ccc; z-index: 10000; font-family: Arial, sans-serif; box-shadow: 0 4px 8px rgba(0,0,0,0.2); overflow: hidden; resize: both; border-radius: 8px; min-width: 300px; min-height: 300px; } #crawler-container.dark { --bg-color: #2b2b2b; color: #f1f1f1; border-color: #555; } #crawler-header { display: flex; justify-content: space-between; align-items: center; background: var(--header-bg, #f1f1f1); border-bottom: 1px solid #ccc; padding: 10px; cursor: move; } #crawler-container.dark #crawler-header { --header-bg: #3c3c3c; border-color: #555; } #crawler-minimize, #dark-mode-toggle { background: none; border: none; font-size: 18px; cursor: pointer; color: inherit; } #crawler-tabs { display: flex; flex-wrap: wrap; gap: 5px; } .crawler-tab { padding: 8px 12px; cursor: pointer; font-weight: bold; border-radius: 4px; } .crawler-tab.active { background: var(--active-tab-bg, #ddd); } #crawler-container.dark .crawler-tab.active { background: #555; } #crawler-content { padding: 10px; overflow-y: auto; height: calc(100% - 60px); } table { width: 100%; border-collapse: collapse; table-layout: fixed; word-wrap: break-word; } th, td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; } th { background-color: var(--th-bg, #f2f2f2); cursor: pointer; } #crawler-container.dark th { background-color: #444; } tr:hover { background-color: #f5f5f5; } #crawler-container.dark tr:hover { background-color: #555; } .progress-bar { width: 100%; background-color: #f3f3f3; border: 1px solid #ccc; border-radius: 5px; overflow: hidden; margin-bottom: 10px; } .progress-bar-fill { height: 20px; background-color: #4caf50; width: 0%; transition: width 0.2s; } .modal { display: none; position: fixed; z-index: 10001; left: 0; top: 0; width: 100%; height: 100%; overflow: auto; background-color: rgba(0,0,0,0.4); } .modal-content { background-color: var(--modal-bg, #fff); margin: 5% auto; padding: 20px; border: 1px solid #888; width: 90%; max-width: 500px; max-height: 80%; overflow-y: auto; border-radius: 8px; } #crawler-container.dark .modal-content { --modal-bg: #3c3c3c; color: #f1f1f1; border-color: #555; } .close { color: #aaa; float: right; font-size: 28px; font-weight: bold; cursor: pointer; } .close:hover, .close:focus { color: black; text-decoration: none; cursor: pointer; } .control-buttons { margin: 10px 0; display: flex; flex-wrap: wrap; gap: 10px; } .control-buttons button { padding: 8px 12px; cursor: pointer; border: none; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px; flex: 1 1 auto; min-width: 80px; } .control-buttons button:disabled { background-color: #a5d6a7; cursor: not-allowed; } .status-text { margin-top: 10px; font-weight: bold; font-size: 14px; } .settings-section { margin-bottom: 15px; } .settings-section label { display: block; margin-bottom: 5px; font-weight: bold; } .settings-section input[type="text"], .settings-section input[type="number"], .settings-section textarea, .settings-section select { width: 100%; padding: 5px; box-sizing: border-box; border: 1px solid #ccc; border-radius: 4px; font-size: 14px; } .settings-section.dark .settings-section input, .settings-section.dark .settings-section select, .settings-section.dark .settings-section textarea { background-color: #555; color: #f1f1f1; border-color: #777; } .checkbox-group, .radio-group { display: flex; flex-wrap: wrap; gap: 10px; } .checkbox-group label, .radio-group label { font-size: 14px; } .tooltip { position: relative; display: inline-block; cursor: help; font-size: 14px; } .tooltip .tooltiptext { visibility: hidden; width: 220px; background-color: #555; color: #fff; text-align: center; border-radius: 6px; padding: 5px; position: absolute; z-index: 1; bottom: 125%; left: 50%; margin-left: -110px; opacity: 0; transition: opacity 0.3s; } .tooltip:hover .tooltiptext { visibility: visible; opacity: 1; } #crawler-container::-webkit-resizer { background: #f1f1f1; border: none; } #data-visualization { margin-top: 20px; } #data-chart { width: 100%; height: 300px; } #modal-html-content { white-space: pre-wrap; word-wrap: break-word; background-color: #f9f9f9; padding: 10px; border-radius: 4px; } #crawler-container.dark #modal-html-content { background-color: #3c3c3c; color: #f1f1f1; } #previous-crawls table, #previous-crawls th, #previous-crawls td { width: 100%; padding: 5px; border: 1px solid #ddd; } #reset-data { background-color: #f44336; } .spinner { border: 4px solid #f3f3f3; border-top: 4px solid #3498db; border-radius: 50%; width: 24px; height: 24px; animation: spin 2s linear infinite; display: inline-block; vertical-align: middle; } @keyframes spin { 0% { transform: rotate(0deg); } 100% { transform: rotate(360deg); } } .error-message { color: red; font-weight: bold; margin-top: 10px; } .success-message { color: green; font-weight: bold; margin-top: 10px; } .dark-mode-toggle { background: none; border: none; cursor: pointer; font-size: 18px; color: inherit; } @media (max-width: 768px) { #crawler-container { width: 100%; height: 100%; top: 0; right: 0; border-radius: 0; } #crawler-header { flex-direction: row; align-items: center; } #crawler-tabs { flex-direction: row; overflow-x: auto; } .crawler-tab { flex: 0 0 auto; } } /* 추가된 CSS: URL 전체 표시 */ td a.full-url { display: block; word-break: break-all; position: relative; } td a.full-url:hover::after { content: attr(href); position: absolute; background: #333; color: #fff; padding: 5px; border-radius: 5px; white-space: nowrap; z-index: 10002; top: 100%; left: 0; } `); const container = $(` <div id="crawler-container"> <div id="crawler-header"> <div id="crawler-tabs"> <div class="crawler-tab" data-tab="logs" title="로그 보기">📜 로그</div> <div class="crawler-tab" data-tab="data" title="수집된 데이터 보기">📊 데이터</div> <div class="crawler-tab" data-tab="settings" title="설정 변경">⚙️ 설정</div> <div class="crawler-tab" data-tab="previous-crawls" title="이전 크롤링 기록">📁 기록</div> <div class="crawler-tab" data-tab="templates" title="설정 템플릿">📂 템플릿</div> <div class="crawler-tab" data-tab="tests" title="테스트 실행">🧪 테스트</div> <button id="dark-mode-toggle" class="dark-mode-toggle" title="테마 전환">🌙</button> </div> <button id="crawler-minimize" title="창 최소화">─</button> </div> <div id="crawler-content"> <div id="tab-logs"> <div class="progress-bar"> <div class="progress-bar-fill" style="width: 0%;"></div> </div> <div class="status-text">진행률: 0% (0/0 페이지)</div> <div class="control-buttons"> <button id="start-crawling" title="크롤링을 시작합니다.">▶ 시작</button> <button id="stop-crawling" title="크롤링을 중지합니다." disabled>■ 중지</button> ${Config.settings.exportFormats.csv ? `<button id="download-csv" title="CSV 다운로드">📥 CSV</button>` : ''} ${Config.settings.exportFormats.json ? `<button id="download-json" title="JSON 다운로드">📥 JSON</button>` : ''} ${Config.settings.exportFormats.xml ? `<button id="download-xml" title="XML 다운로드">📥 XML</button>` : ''} ${Config.settings.exportFormats.xls ? `<button id="download-xls" title="XLS 다운로드">📥 XLS</button>` : ''} <button id="reset-data" title="모든 크롤링 데이터를 초기화합니다.">🗑️ 초기화</button> </div> <div id="log-messages" style="height: 200px; overflow-y: auto; background: var(--log-bg, #f9f9f9); padding: 10px; border: 1px solid #ccc; border-radius: 4px;"> 로그 내용이 여기에 표시됩니다. </div> <div id="error-message" class="error-message" style="display: none;"></div> <div id="success-message" class="success-message" style="display: none;"></div> </div> <div id="tab-data" style="display: none;"> <input type="text" id="search-filter" placeholder="검색 필터..." style="width: 100%; padding: 5px; margin-bottom: 10px; border: 1px solid #ccc; border-radius: 4px;"> <table> <thead> <tr> ${Config.settings.dataFields.name ? '<th data-sort="name">사이트 이름</th>' : ''} ${Config.settings.dataFields.url ? '<th data-sort="url">주소</th>' : ''} ${Config.settings.dataFields.description ? '<th data-sort="description">설명</th>' : ''} ${Config.settings.dataFields.html ? '<th>📄 HTML 내용</th>' : ''} ${Config.settings.dataFields.category ? '<th data-sort="category">분류</th>' : ''} ${Config.settings.dataFields.quality ? '<th data-sort="quality">품질점수</th>' : ''} ${Config.settings.dataFields.language ? '<th data-sort="language">언어</th>' : ''} ${Config.settings.dataFields.wordCount ? '<th data-sort="wordCount">단어 수</th>' : ''} ${Config.settings.dataFields.metaTags ? '<th>메타 태그</th>' : ''} ${Config.settings.dataFields.headings ? '<th>헤딩</th>' : ''} </tr> </thead> <tbody id="data-table-body"></tbody> </table> ${Config.settings.dataVisualization ? ` <div id="data-visualization"> <h3>데이터 시각화</h3> <canvas id="data-chart"></canvas> </div> ` : ''} </div> <div id="tab-settings" style="display: none;"> <h3>설정</h3> ${createSettingsSections()} <div class="settings-section"> <button id="save-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">저장</button> <button id="reset-settings" style="margin-top: 10px; padding: 10px 20px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">초기화</button> </div> </div> <div id="tab-templates" style="display: none;"> <h3>설정 템플릿</h3> <div class="settings-section"> <label for="new-template-name">새 템플릿 이름:</label> <input type="text" id="new-template-name" placeholder="템플릿 이름을 입력하세요"> <button id="save-template" style="margin-top: 5px; padding: 8px 12px; background-color: #4caf50; color: white; border-radius: 4px; font-size: 14px;">📁 템플릿 저장</button> </div> <div class="settings-section"> <label for="template-select">기존 템플릿:</label> <select id="template-select" style="width: 100%; padding: 5px; margin-top: 10px; border: 1px solid #ccc; border-radius: 4px;"> <option value="" disabled selected>템플릿 선택</option> ${Object.keys(Config.settings.templates).map(t => `<option value="${t}">${t}</option>`).join('')} </select> <button id="load-template" style="margin-top: 10px; padding: 8px 12px; background-color: #2196F3; color: white; border-radius: 4px; font-size: 14px;">📂 템플릿 불러오기</button> <button id="delete-template" style="margin-top: 5px; padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px;">템플릿 삭제</button> </div> </div> <div id="tab-previous-crawls" style="display: none;"> <h3>이전 크롤링 기록</h3> <button id="clear-previous-crawls" style="padding: 8px 12px; background-color: #f44336; color: white; border-radius: 4px; font-size: 14px; margin-bottom: 10px;">기록 초기화</button> <table> <thead> <tr> <th>날짜</th> <th>크롤링 수</th> <th>상태</th> </tr> </thead> <tbody id="previous-crawls-body"></tbody> </table> </div> <div id="tab-tests" style="display: none;"> <button id="run-tests" style="padding: 10px 20px; background-color: #FFC107; color: white; border-radius: 4px; font-size: 14px;">테스트 실행</button> </div> <div id="html-modal" class="modal"> <div class="modal-content"> <span class="close">×</span> <pre><code id="modal-html-content" class="hljs language-html"></code></pre> </div> </div> <div id="test-modal" class="modal"> <div class="modal-content"> <span class="close">×</span> <div id="mocha"></div> </div> </div> </div> </div> `); $('body').append(container); bindEvents(); makeDraggable(); } function createSettingsSections() { const sections = [ { label: '크롤링 딜레이 (ms)', id: 'delay-input', type: 'number', value: Config.settings.delay, min: 500, step: 500, tooltip: '기본 딜레이과 딜레이 랜덤 값을 조합하여 요청 사이의 딜레이를 설정합니다.' }, { label: '딜레이 랜덤 범위 (ms)', id: 'delay-random-input', type: 'number', value: Config.settings.delayRandom, min: 0, step: 500, tooltip: '딜레이에 무작위 요소를 추가하여 보다 자연스러운 요청을 합니다.' }, { label: '최대 크롤링 페이지 수', id: 'max-pages-input', type: 'number', value: Config.settings.maxPages, min: 1, step: 1, tooltip: '크롤링할 최대 페이지 수를 설정하세요.' }, { label: '사용자 에이전트(User-Agent)', id: 'user-agent-select', type: 'select-multiple', options: Config.settings.userAgents.map((ua, idx) => ({ value: idx, text: ua, selected: idx === Config.settings.userAgentIndex })), buttons: [ { id: 'add-user-agent', text: '추가', bgColor: '#2196F3' }, { id: 'remove-user-agent', text: '제거', bgColor: '#f44336' } ], tooltip: '크롤링에 사용할 사용자 에이전트를 선택하거나 추가할 수 있습니다.' }, { label: '콘텐츠 유형 포함', id: 'content-include-input', type: 'text', value: Config.settings.contentTypes.include.join(', '), tooltip: '크롤링할 콘텐츠 유형을 쉼표로 구분하여 입력하세요.' }, { label: '콘텐츠 유형 제외', id: 'content-exclude-input', type: 'text', value: Config.settings.contentTypes.exclude.join(', '), tooltip: '크롤링에서 제외할 콘텐츠 유형을 쉼표로 구분하여 입력하세요.' }, { label: '크롤링 타임아웃 (ms)', id: 'timeout-input', type: 'number', value: Config.settings.timeout, min: 1000, step: 1000, tooltip: '요청이 완료되기까지 기다릴 최대 시간을 밀리초 단위로 설정합니다.' }, { label: '포함할 키워드', id: 'include-keywords-input', type: 'textarea', value: Config.settings.includeKeywords.join('\n'), tooltip: '크롤링할 페이지에 반드시 포함되어야 하는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.' }, { label: '제외할 키워드', id: 'exclude-keywords-input', type: 'textarea', value: Config.settings.excludeKeywords.join('\n'), tooltip: '크롤링할 페이지에 포함되면 안 되는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.' }, { label: '포함할 키워드 (정규 표현식)', id: 'regex-include-input', type: 'textarea', value: Config.settings.regexInclude.join('\n'), tooltip: '포함할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.' }, { label: '제외할 키워드 (정규 표현식)', id: 'regex-exclude-input', type: 'textarea', value: Config.settings.regexExclude.join('\n'), tooltip: '제외할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.' }, { label: '언어 필터링', id: 'language-select', type: 'select-multiple', options: [ { value: 'en', text: '영어 (en)' }, { value: 'ko', text: '한국어 (ko)' }, { value: 'zh', text: '중국어 (zh)' }, { value: 'ja', text: '일본어 (ja)' }, { value: 'ru', text: '러시아어 (ru)' }, { value: 'es', text: '스페인어 (es)' }, { value: 'fr', text: '프랑스어 (fr)' }, { value: 'de', text: '독일어 (de)' }, { value: 'it', text: '이탈리아어 (it)' }, { value: 'pt', text: '포르투갈어 (pt)' }, { value: 'ar', text: '아랍어 (ar)' }, { value: 'hi', text: '힌디어 (hi)' }, { value: 'bn', text: '벵골어 (bn)' }, { value: 'pa', text: '펀자브어 (pa)' }, { value: 'nl', text: '네덜란드어 (nl)' }, { value: 'sv', text: '스웨덴어 (sv)' }, { value: 'tr', text: '터키어 (tr)' }, { value: 'vi', text: '베트남어 (vi)' }, { value: 'pl', text: '폴란드어 (pl)' }, { value: 'th', text: '태국어 (th)' } ].map(lang => ({ value: lang.value, text: lang.text, selected: Config.settings.language.includes(lang.value) })) }, { label: '최소 단어 수', id: 'min-word-count-input', type: 'number', value: Config.settings.minWordCount, min: 10, step: 10, tooltip: '수집할 페이지의 최소 단어 수를 설정하세요.' }, { label: '최대 단어 수', id: 'max-word-count-input', type: 'number', value: Config.settings.maxWordCount, min: 100, step: 100, tooltip: '수집할 페이지의 최대 단어 수를 설정하세요.' }, { label: '최대 크롤링 깊이', id: 'max-depth-input', type: 'number', value: Config.settings.maxDepth, min: 1, step: 1, tooltip: '링크를 추적할 최대 깊이를 설정하세요.' }, { label: '프록시 서버', id: 'proxy-input', type: 'text', value: Config.settings.proxy, placeholder: 'http://proxyserver:port', tooltip: '프록시 서버 주소를 입력하세요. 예: http://proxyserver:port' }, { label: '수집할 데이터 필드', id: '', type: 'checkbox-group', options: Object.keys(Config.settings.dataFields).map(field => ({ id: `field-${field}`, text: Utils.capitalize(getFieldName(field)), checked: Config.settings.dataFields[field] })) }, { label: '재시도 횟수', id: 'retry-attempts-input', type: 'number', value: Config.settings.retryAttempts, min: 0, step: 1, tooltip: '요청 실패 시 재시도할 횟수를 설정하세요.' }, { label: '재시도 딜레이 (ms)', id: 'retry-delay-input', type: 'number', value: Config.settings.retryDelay, min: 1000, step: 500, tooltip: '재시도 시 대기할 시간을 밀리초 단위로 설정하세요.' }, { label: '크롤링 스케줄링', id: 'schedule-enabled', type: 'checkbox', checked: Config.settings.schedule.enabled, tooltip: '크롤링 스케줄링을 활성화합니다.' }, { label: '시작 시간 (HH:MM)', id: 'start-time-input', type: 'time', value: Config.settings.schedule.startTime, display: Config.settings.schedule.enabled, tooltip: '크롤링을 시작할 시간을 설정하세요.' }, { label: '크롤링 간격 (분)', id: 'interval-input', type: 'number', value: Config.settings.schedule.interval / 60000, min: 1, step: 1, display: Config.settings.schedule.enabled, tooltip: '크롤링 작업 간의 간격을 분 단위로 설정하세요.' }, { label: '특정 요일', id: 'specific-days-input', type: 'select-multiple', options: [ { value: '0', text: '일요일' }, { value: '1', text: '월요일' }, { value: '2', text: '화요일' }, { value: '3', text: '수요일' }, { value: '4', text: '목요일' }, { value: '5', text: '금요일' }, { value: '6', text: '토요일' } ].map(day => ({ value: day.value, text: day.text, selected: Config.settings.schedule.specificDays.includes(parseInt(day.value)) })), display: Config.settings.schedule.enabled, tooltip: '크롤링을 수행할 특정 요일을 선택하세요. (복수 선택 가능)' }, { label: '크롤링할 검색 엔진 선택', id: '', type: 'checkbox-group', options: Object.keys(Config.settings.searchEngines).map(engine => ({ id: `engine-${engine}`, text: Utils.capitalize(engine), checked: Config.settings.searchEngines[engine] })) }, { label: '블랙리스트 도메인', id: 'blacklist-input', type: 'textarea', value: Config.settings.blacklist.join('\n'), tooltip: '크롤링에서 제외할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.' }, { label: '화이트리스트 도메인', id: 'whitelist-input', type: 'textarea', value: Config.settings.whitelist.join('\n'), tooltip: '크롤링을 허용할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.' }, { label: 'robots.txt 준수', id: 'respect-robots-checkbox', type: 'checkbox', checked: Config.settings.respectRobots, tooltip: '대상 사이트의 robots.txt 규칙을 준수하여 크롤링을 진행합니다.' }, { label: '동시 요청 최대 수', id: 'max-concurrent-requests-input', type: 'number', value: Config.settings.maxConcurrentRequests, min: 1, step: 1, tooltip: '한 번에 요청할 수 있는 최대 동시 요청 수를 설정하세요.' }, { label: '데이터 시각화 활성화', id: 'data-visualization-checkbox', type: 'checkbox', checked: Config.settings.dataVisualization, tooltip: '수집된 데이터를 시각화하여 그래프로 표시합니다.' }, { label: 'CAPTCHA 처리 활성화', id: 'enable-captchas-checkbox', type: 'checkbox', checked: Config.settings.enableCaptchas, tooltip: 'CAPTCHA가 감지되었을 때 자동 처리를 활성화합니다. (제한적)' }, { label: '데이터 내보내기 형식', id: '', type: 'checkbox-group', options: Object.keys(Config.settings.exportFormats).map(format => ({ id: `export-${format}`, text: format.toUpperCase(), checked: Config.settings.exportFormats[format] })) }, { label: '고급 로깅 활성화', id: 'advanced-logging-checkbox', type: 'checkbox', checked: Config.settings.advancedLogging, tooltip: '추가적인 로깅 정보를 활성화합니다.' }, { label: '사용자 지정 키워드 분류', id: 'custom-keywords-input', type: 'textarea', value: Utils.formatCustomKeywords(Config.settings.customKeywords), tooltip: '사용자 지정 키워드를 추가하여 데이터 분류를 향상시킵니다. 형식: 카테고리:키워드1,키워드2,...' } ]; return sections.map(sec => { if (sec.type === 'checkbox-group') { return ` <div class="settings-section"> <label>${sec.label}:</label> <div class="checkbox-group"> ${sec.options.map(opt => `<label><input type="checkbox" id="${opt.id}" ${opt.checked ? 'checked' : ''}> ${opt.text}</label>`).join('')} </div> </div> `; } if (sec.type === 'select-multiple') { return ` <div class="settings-section"> <label for="${sec.id}">${sec.label}: <span class="tooltip">ℹ️ <span class="tooltiptext">${sec.tooltip}</span> </span> </label> <select id="${sec.id}" multiple size="${sec.options.length || 5}"> ${sec.options.map(opt => `<option value="${opt.value}" ${opt.selected ? 'selected' : ''}>${opt.text}</option>`).join('')} </select> </div> `; } if (sec.type === 'checkbox') { return ` <div class="settings-section"> <label for="${sec.id}">${sec.label}: <span class="tooltip">ℹ️ <span class="tooltiptext">${sec.tooltip}</span> </span> </label> <input type="checkbox" id="${sec.id}" ${sec.checked ? 'checked' : ''}> </div> `; } if (sec.type === 'textarea') { return ` <div class="settings-section"> <label for="${sec.id}">${sec.label}: <span class="tooltip">ℹ️ <span class="tooltiptext">${sec.tooltip}</span> </span> </label> <textarea id="${sec.id}" rows="3">${sec.value}</textarea> </div> `; } if (sec.type === 'select') { return ` <div class="settings-section"> <label for="${sec.id}">${sec.label}:</label> <select id="${sec.id}" ${sec.multiple ? 'multiple' : ''}> ${sec.options.map(opt => `<option value="${opt.value}" ${opt.selected ? 'selected' : ''}>${opt.text}</option>`).join('')} </select> </div> `; } return ` <div class="settings-section"> <label for="${sec.id}">${sec.label}: <span class="tooltip">ℹ️ <span class="tooltiptext">${sec.tooltip}</span> </span> </label> <input type="${sec.type}" id="${sec.id}" value="${sec.value}" ${sec.min ? `min="${sec.min}"` : ''} ${sec.step ? `step="${sec.step}"` : ''} ${sec.placeholder ? `placeholder="${sec.placeholder}"` : ''} ${sec.display === false ? 'style="display: none;"' : ''}> ${sec.buttons ? sec.buttons.map(btn => `<button id="${btn.id}" style="margin-top: 5px; padding: 5px 10px; background-color: ${btn.bgColor}; color: white; border-radius: 4px; font-size: 12px;">${btn.text}</button>`).join('') : ''} </div> `; }).join(''); } function bindEvents() { $('#crawler-minimize').on('click', toggleMinimize); $('.crawler-tab').on('click', switchTab); $('#save-settings').on('click', saveSettingsHandler); $('#reset-settings').on('click', resetSettingsHandler); $('#add-user-agent').on('click', addUserAgent); $('#remove-user-agent').on('click', removeUserAgent); $('#download-csv').on('click', Download.downloadCSV); $('#download-json').on('click', Download.downloadJSON); $('#download-xml').on('click', Download.downloadXML); $('#download-xls').on('click', Download.downloadXLS); $('#search-filter').on('input', filterData); $('#schedule-enabled').on('change', toggleScheduleOptions); $('#data-visualization-checkbox').on('change', toggleDataVisualization); $('#enable-captchas-checkbox').on('change', toggleCaptchaHandling); $('#save-template').on('click', saveTemplateHandler); $('#load-template').on('click', loadTemplateHandler); $('#delete-template').on('click', deleteTemplateHandler); $('#template-select').on('change', selectTemplateHandler); $('#run-tests').on('click', Testing.runTests); $('#clear-previous-crawls').on('click', clearPreviousCrawls); $('#reset-data').on('click', resetData); $('#dark-mode-toggle').on('click', toggleDarkMode); $('.close').on('click', () => $('.modal').hide()); $(window).on('click', e => { if ($(e.target).hasClass('modal')) $('.modal').hide(); }); $('#start-crawling').on('click', Crawling.startCrawling); $('#stop-crawling').on('click', Crawling.stopCrawling); $(document).on('keydown', handleShortcuts); $('#proxy-input').on('input', validateProxyInput); $('input[type="checkbox"], input[type="radio"]').on('change', toggleDataField); } function toggleMinimize() { $('#crawler-content').toggle(); $('#crawler-minimize').text($('#crawler-content').is(':hidden') ? '☐' : '─'); Logger.log(`크롤러 창이 ${$('#crawler-content').is(':hidden') ? '최소화' : '확대'}되었습니다.`, 'info'); } function switchTab() { const tab = $(this).data('tab'); $('.crawler-tab').removeClass('active'); $(this).addClass('active'); $('#crawler-content > div').hide(); $(`#tab-${tab}`).show(); Config.saveSettings({ currentTab: tab }); } function updateUI() { const currentTab = Config.settings.currentTab || 'logs'; $(`.crawler-tab[data-tab="${currentTab}"]`).addClass('active'); $('#crawler-content > div').hide(); $(`#tab-${currentTab}`).show(); updateLogs(); updateDataTable(); updatePreviousCrawlsUI(); updateTemplatesUI(); if (Config.settings.dataVisualization) DataVisualization.renderChart(); applyDarkMode(); } function updateLogs() { const logs = Logger.getLogs(); $('#log-messages').html(logs.map(l => `<div>[${l.timestamp}] [${l.type.toUpperCase()}] ${Utils.escape(l.message)}</div>`).join('')); const progress = GM_getValue('progress', 0); const currentPage = GM_getValue('currentPage', 0); const totalPages = Config.settings.maxPages; $('.progress-bar-fill').css('width', `${progress.toFixed(2)}%`); $('.status-text').text(`진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)`); const latestLog = logs[logs.length -1]; if (latestLog) { if (['error', 'warning'].includes(latestLog.type)) { $('#error-message').text(latestLog.message).show().delay(5000).fadeOut(); } if (['success', 'info'].includes(latestLog.type)) { $('#success-message').text(latestLog.message).show().delay(5000).fadeOut(); } } } function updateDataTable() { const data = JSON.parse(GM_getValue('data', '[]')); const filter = $('#search-filter').val().toLowerCase(); const filtered = filter ? data.filter(item => Object.keys(Config.settings.dataFields).some(f => Config.settings.dataFields[f] && String(item[f]).toLowerCase().includes(filter))) : data; $('#data-table-body').html(filtered.map(item => ` <tr> ${Config.settings.dataFields.name ? `<td>${Utils.escape(item.name)}</td>` : ''} ${Config.settings.dataFields.url ? `<td><a href="${Utils.escape(item.url)}" target="_blank" class="full-url">${Utils.escape(item.url)}</a></td>` : ''} ${Config.settings.dataFields.description ? `<td>${Utils.escape(item.description)}</td>` : ''} ${Config.settings.dataFields.html ? `<td><button class="view-html" data-html="${encodeURIComponent(item.html)}">📄 보기</button></td>` : ''} ${Config.settings.dataFields.category ? `<td>${Utils.escape(item.category)}</td>` : ''} ${Config.settings.dataFields.quality ? `<td>${Utils.escape(item.quality.toString())}</td>` : ''} ${Config.settings.dataFields.language ? `<td>${Utils.escape(item.language)}</td>` : ''} ${Config.settings.dataFields.wordCount ? `<td>${Utils.escape(item.wordCount.toString())}</td>` : ''} ${Config.settings.dataFields.metaTags ? `<td>${Utils.escape(JSON.stringify(item.metaTags))}</td>` : ''} ${Config.settings.dataFields.headings ? `<td>${Utils.escape(item.headings.join(', '))}</td>` : ''} </tr> `).join('')); $('.view-html').off('click').on('click', function() { const htmlContent = decodeURIComponent($(this).data('html')); $('#modal-html-content').text(htmlContent); const codeBlock = document.getElementById('modal-html-content'); hljs.highlightElement(codeBlock); $('#html-modal').show(); }); if (Config.settings.dataVisualization) DataVisualization.renderChart(); } function updatePreviousCrawlsUI() { const previousCrawls = JSON.parse(GM_getValue('previousCrawls', '[]')); $('#previous-crawls-body').html(previousCrawls.map(crawl => ` <tr> <td>${Utils.escape(crawl.date)}</td> <td>${Utils.escape(crawl.count.toString())}</td> <td>${Utils.escape(crawl.status)}</td> </tr> `).join('')); } function updateTemplatesUI() { $('#template-select').empty().append(`<option value="" disabled selected>템플릿 선택</option>`); Object.keys(Config.settings.templates).forEach(t => $('#template-select').append(`<option value="${t}">${t}</option>`)); } function addUserAgent() { const ua = prompt("새 사용자 에이전트를 입력하세요:"); if (ua) { Config.settings.userAgents.push(ua); Config.saveSettings({ userAgents: Config.settings.userAgents }); updateUserAgentSelect(); Logger.log('새 사용자 에이전트가 추가되었습니다.', 'success'); } } function removeUserAgent() { const selected = $('#user-agent-select').val(); if (selected && selected.length) { selected.sort((a, b) => b - a).forEach(idx => Config.settings.userAgents.splice(idx, 1)); Config.settings.userAgentIndex = 0; Config.saveSettings({ userAgents: Config.settings.userAgents, userAgentIndex: Config.settings.userAgentIndex }); updateUserAgentSelect(); Logger.log('선택한 사용자 에이전트가 제거되었습니다.', 'success'); } else { alert('제거할 사용자 에이전트를 선택하세요.'); } } function updateUserAgentSelect() { $('#user-agent-select').empty(); Config.settings.userAgents.forEach((ua, idx) => { $('#user-agent-select').append(`<option value="${idx}" ${idx === Config.settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`); }); } function saveSettingsHandler() { const newSettings = { delay: parseInt($('#delay-input').val(), 10) || Config.DEFAULT_SETTINGS.delay, delayRandom: parseInt($('#delay-random-input').val(), 10) || Config.DEFAULT_SETTINGS.delayRandom, maxPages: parseInt($('#max-pages-input').val(), 10) || Config.DEFAULT_SETTINGS.maxPages, userAgentIndex: parseInt($('#user-agent-select').val(), 10) || 0, contentTypes: { include: $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(Boolean), exclude: $('#content-exclude-input').val().split(',').map(ct => ct.trim()).filter(Boolean) }, timeout: parseInt($('#timeout-input').val(), 10) || Config.DEFAULT_SETTINGS.timeout, includeKeywords: $('#include-keywords-input').val().split('\n').map(k => k.trim()).filter(Boolean), excludeKeywords: $('#exclude-keywords-input').val().split('\n').map(k => k.trim()).filter(Boolean), regexInclude: $('#regex-include-input').val().split('\n').map(r => r.trim()).filter(Boolean), regexExclude: $('#regex-exclude-input').val().split('\n').map(r => r.trim()).filter(Boolean), language: $('#language-select').val() || [], minWordCount: parseInt($('#min-word-count-input').val(), 10) || Config.DEFAULT_SETTINGS.minWordCount, maxWordCount: parseInt($('#max-word-count-input').val(), 10) || Config.DEFAULT_SETTINGS.maxWordCount, maxDepth: parseInt($('#max-depth-input').val(), 10) || Config.DEFAULT_SETTINGS.maxDepth, proxy: $('#proxy-input').val().trim(), dataFields: Object.fromEntries(['name','url','description','html','category','quality','language','wordCount','metaTags','headings'].map(f => [`${f}`, $(`#field-${f}`).is(':checked') ])), retryAttempts: parseInt($('#retry-attempts-input').val(), 10) || Config.DEFAULT_SETTINGS.retryAttempts, retryDelay: parseInt($('#retry-delay-input').val(), 10) || Config.DEFAULT_SETTINGS.retryDelay, schedule: { enabled: $('#schedule-enabled').is(':checked'), startTime: $('#start-time-input').val() || Config.DEFAULT_SETTINGS.schedule.startTime, interval: (parseInt($('#interval-input').val(), 10) || (Config.DEFAULT_SETTINGS.schedule.interval / 60000)) * 60000, specificDays: $('#specific-days-input').val() ? $('#specific-days-input').val().map(d => parseInt(d, 10)) : [] }, searchEngines: Object.fromEntries(Object.keys(Config.settings.searchEngines).map(engine => [`${engine}`, $(`#engine-${engine}`).is(':checked') ])), blacklist: $('#blacklist-input').val().split('\n').map(d => d.trim()).filter(Boolean), whitelist: $('#whitelist-input').val().split('\n').map(d => d.trim()).filter(Boolean), respectRobots: $('#respect-robots-checkbox').is(':checked'), maxConcurrentRequests: parseInt($('#max-concurrent-requests-input').val(), 10) || Config.DEFAULT_SETTINGS.maxConcurrentRequests, dataVisualization: $('#data-visualization-checkbox').is(':checked'), exportFormats: Object.fromEntries(['csv','json','xml','xls'].map(fmt => [`${fmt}`, $(`#export-${fmt}`).is(':checked') ])), advancedLogging: $('#advanced-logging-checkbox').is(':checked'), enableCaptchas: $('#enable-captchas-checkbox').is(':checked'), customKeywords: Utils.parseCustomKeywords($('#custom-keywords-input').val()) }; if (newSettings.proxy && !Config.validateProxy(newSettings.proxy)) { $('#error-message').text('유효하지 않은 프록시 URL입니다.').show(); setTimeout(() => $('#error-message').fadeOut(), 5000); return; } Config.saveSettings(newSettings); Crawling.updateCrawlSettings(); Logger.log('설정이 저장되었습니다.', 'success'); if (newSettings.schedule.enabled) Scheduler.scheduleCrawling(); updateDataTable(); if (newSettings.dataVisualization) DataVisualization.renderChart(); } function resetSettingsHandler() { if (confirm("기본 설정으로 초기화하시겠습니까?")) { Config.resetSettings(); location.reload(); } } function toggleScheduleOptions() { $('#start-time-input, #interval-input, #specific-days-input').toggle($('#schedule-enabled').is(':checked')); } function toggleDataVisualization() { const enabled = $('#data-visualization-checkbox').is(':checked'); Config.saveSettings({ dataVisualization: enabled }); $('#data-visualization').toggle(enabled); Logger.log(`데이터 시각화가 ${enabled ? '활성화' : '비활성화'}되었습니다.`, 'success'); if (enabled) DataVisualization.renderChart(); } function toggleCaptchaHandling() { const enabled = $('#enable-captchas-checkbox').is(':checked'); Config.saveSettings({ enableCaptchas: enabled }); Logger.log(`CAPTCHA 처리 ${enabled ? '활성화' : '비활성화'}되었습니다.`, 'success'); } function filterData() { updateDataTable(); } function clearPreviousCrawls() { if (confirm("이전 크롤링 기록을 모두 초기화하시겠습니까?")) { GM_setValue('previousCrawls', JSON.stringify([])); updatePreviousCrawlsUI(); Logger.log('이전 크롤링 기록이 초기화되었습니다.', 'success'); } } function resetData() { if (confirm("모든 크롤링 데이터를 초기화하시겠습니까?")) { GM_setValue('data', JSON.stringify([])); GM_setValue('visitedUrls', JSON.stringify([])); updateDataTable(); Logger.log('크롤링 데이터가 초기화되었습니다.', 'success'); } } function handleShortcuts(e) { if (e.ctrlKey) { const actions = { '1': 'logs', '2': 'data', '3': 'settings', '4': 'previous-crawls', '5': 'templates', '6': 'tests', 's': 'save', 'r': 'reset', 'p': 'start', 'q': 'stop', 'c': 'csv', 'j': 'json', 'x': 'xml', 'l': 'xls', 'd': 'dark-mode' }; const action = actions[e.key]; if (action) { e.preventDefault(); switch(action) { case 'logs': $('.crawler-tab[data-tab="logs"]').click(); break; case 'data': $('.crawler-tab[data-tab="data"]').click(); break; case 'settings': $('.crawler-tab[data-tab="settings"]').click(); break; case 'previous-crawls': $('.crawler-tab[data-tab="previous-crawls"]').click(); break; case 'templates': $('.crawler-tab[data-tab="templates"]').click(); break; case 'tests': $('.crawler-tab[data-tab="tests"]').click(); break; case 'save': saveSettingsHandler(); break; case 'reset': resetSettingsHandler(); break; case 'start': if (!Crawling.isCrawling()) Crawling.startCrawling(); break; case 'stop': if (Crawling.isCrawling()) Crawling.stopCrawling(); break; case 'csv': Config.settings.exportFormats.csv && Download.downloadCSV(); break; case 'json': Config.settings.exportFormats.json && Download.downloadJSON(); break; case 'xml': Config.settings.exportFormats.xml && Download.downloadXML(); break; case 'xls': Config.settings.exportFormats.xls && Download.downloadXLS(); break; case 'dark-mode': toggleDarkMode(); break; } } } } function makeDraggable() { const container = document.getElementById('crawler-container'); const header = document.getElementById('crawler-header'); let isDragging = false, startX, startY, initialX, initialY; header.onmousedown = function(e) { isDragging = true; startX = e.clientX; startY = e.clientY; initialX = container.offsetLeft; initialY = container.offsetTop; document.onmousemove = drag; document.onmouseup = stopDrag; return false; }; function drag(e) { if (isDragging) { container.style.left = (initialX + e.clientX - startX) + 'px'; container.style.top = (initialY + e.clientY - startY) + 'px'; } } function stopDrag() { isDragging = false; document.onmousemove = null; document.onmouseup = null; } } function getFieldName(field) { const names = { name: '이름', url: '주소', description: '설명', html: 'HTML 내용', category: '분류', quality: '품질점수', language: '언어', wordCount: '단어 수', metaTags: '메타 태그', headings: '헤딩' }; return names[field] || field; } function saveTemplateHandler() { const templateName = $('#new-template-name').val().trim(); if (!templateName) { alert('템플릿 이름을 입력하세요.'); return; } Config.saveTemplate(templateName); updateTemplatesUI(); $('#new-template-name').val(''); } function loadTemplateHandler() { const selected = $('#template-select').val(); if (!selected) { alert('불러올 템플릿을 선택하세요.'); return; } Config.loadTemplate(selected); updateUI(); $('.crawler-tab[data-tab="settings"]').click(); } function deleteTemplateHandler() { const selected = $('#template-select').val(); if (!selected) { alert('삭제할 템플릿을 선택하세요.'); return; } delete Config.settings.templates[selected]; Config.saveSettings({ templates: Config.settings.templates }); Logger.log(`템플릿 ${selected}이(가) 삭제되었습니다.`, 'success'); updateTemplatesUI(); } function selectTemplateHandler() { const selected = $(this).val(); if (selected) { Config.loadTemplate(selected); updateUI(); $('.crawler-tab[data-tab="settings"]').click(); } } function toggleDarkMode() { $('#crawler-container').toggleClass('dark'); const isDark = $('#crawler-container').hasClass('dark'); $('#dark-mode-toggle').text(isDark ? '☀️' : '🌙'); Config.saveSettings({ darkMode: isDark }); Logger.log(`다크 모드가 ${isDark ? '활성화' : '비활성화'}되었습니다.`, 'success'); } function applyDarkMode() { if (Config.settings.darkMode) { $('#crawler-container').addClass('dark'); $('#dark-mode-toggle').text('☀️'); } else { $('#crawler-container').removeClass('dark'); $('#dark-mode-toggle').text('🌙'); } } function toggleDataField() { const field = $(this).attr('id').replace('field-', ''); Config.settings.dataFields[field] = $(this).is(':checked'); Config.saveSettings({ dataFields: Config.settings.dataFields }); Download.updateExportButtons(); } function validateProxyInput() { const proxy = $('#proxy-input').val().trim(); if (proxy && !Config.validateProxy(proxy)) { $('#error-message').text('유효하지 않은 프록시 URL입니다.').show(); } else { $('#error-message').hide(); } } return { createUI, updateUI, updateLogs, updateDataTable, updatePreviousCrawlsUI, updateTemplatesUI, applyDarkMode }; })(); const Download = (() => { function downloadCSV() { const data = JSON.parse(GM_getValue('data', '[]')); if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; } const headers = Object.keys(Config.settings.dataFields).filter(f => Config.settings.dataFields[f]).map(f => Utils.capitalize(getFieldName(f))); const csvContent = [headers.join(",")] .concat(data.map(item => Object.keys(Config.settings.dataFields).filter(f => Config.settings.dataFields[f]).map(f => `"${Utils.escape(item[f], 'csv')}"`).join(","))) .join("\n"); const blob = new Blob([csvContent], { type: "text/csv;charset=utf-8;" }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); link.download = "data.csv"; document.body.appendChild(link); link.click(); document.body.removeChild(link); Logger.log('CSV 파일이 다운로드되었습니다.', 'success'); } function downloadJSON() { const data = JSON.parse(GM_getValue('data', '[]')); if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; } const jsonContent = JSON.stringify(data, null, 2); const blob = new Blob([jsonContent], { type: "application/json" }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); link.download = "data.json"; document.body.appendChild(link); link.click(); document.body.removeChild(link); Logger.log('JSON 파일이 다운로드되었습니다.', 'success'); } function downloadXML() { const data = JSON.parse(GM_getValue('data', '[]')); if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; } let xmlContent = '<?xml version="1.0" encoding="UTF-8"?><data>'; data.forEach(item => { xmlContent += '<entry>'; if (Config.settings.dataFields.name) xmlContent += `<name>${Utils.escape(item.name || 'No Title', 'xml')}</name>`; if (Config.settings.dataFields.url) xmlContent += `<url>${Utils.escape(item.url || '', 'xml')}</url>`; if (Config.settings.dataFields.description) xmlContent += `<description>${Utils.escape(item.description || 'No Description', 'xml')}</description>`; if (Config.settings.dataFields.html) xmlContent += `<html><![CDATA[${item.html || ''}]]></html>`; if (Config.settings.dataFields.category) xmlContent += `<category>${Utils.escape(item.category || '', 'xml')}</category>`; if (Config.settings.dataFields.quality) xmlContent += `<quality>${Utils.escape(item.quality)}</quality>`; if (Config.settings.dataFields.language) xmlContent += `<language>${Utils.escape(item.language || 'unknown', 'xml')}</language>`; if (Config.settings.dataFields.wordCount) xmlContent += `<wordCount>${Utils.escape(item.wordCount)}</wordCount>`; if (Config.settings.dataFields.metaTags) xmlContent += `<metaTags>${Utils.escape(JSON.stringify(item.metaTags), 'xml')}</metaTags>`; if (Config.settings.dataFields.headings) xmlContent += `<headings>${Utils.escape(item.headings.join('; '), 'xml')}</headings>`; xmlContent += '</entry>'; }); xmlContent += '</data>'; const blob = new Blob([xmlContent], { type: "application/xml" }); const link = document.createElement("a"); link.href = URL.createObjectURL(blob); link.download = "data.xml"; document.body.appendChild(link); link.click(); document.body.removeChild(link); Logger.log('XML 파일이 다운로드되었습니다.', 'success'); } function downloadXLS() { const data = JSON.parse(GM_getValue('data', '[]')); if (!data.length) { Logger.log('다운로드할 데이터가 없습니다.', 'warning'); return; } const worksheetData = data.map(item => { const entry = {}; Object.keys(Config.settings.dataFields).forEach(f => { if (Config.settings.dataFields[f]) { if (f === 'metaTags') entry['메타 태그'] = JSON.stringify(item[f]); else if (f === 'headings') entry['헤딩'] = item[f].join('; '); else entry[Utils.capitalize(getFieldName(f))] = item[f]; } }); return entry; }); const worksheet = XLSX.utils.json_to_sheet(worksheetData); const workbook = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(workbook, worksheet, "Data"); XLSX.writeFile(workbook, "data.xlsx"); Logger.log('XLS 파일이 다운로드되었습니다.', 'success'); } function getFieldName(field) { const names = { name: '이름', url: '주소', description: '설명', html: 'HTML 내용', category: '분류', quality: '품질점수', language: '언어', wordCount: '단어 수', metaTags: '메타 태그', headings: '헤딩' }; return names[field] || field; } function updateExportButtons() { // Placeholder for dynamic enabling/disabling of export buttons based on data fields } return { downloadCSV, downloadJSON, downloadXML, downloadXLS, updateExportButtons }; })(); const DataVisualization = (() => { let chartInstance = null; function renderChart() { const ctx = document.getElementById('data-chart').getContext('2d'); if (chartInstance) chartInstance.destroy(); const data = JSON.parse(GM_getValue('data', '[]')); const categories = data.reduce((acc, item) => { const cat = item.category || '기타'; acc[cat] = (acc[cat] || 0) + 1; return acc; }, {}); const chartData = { labels: Object.keys(categories), datasets: [{ label: '카테고리별 데이터 수', data: Object.values(categories), backgroundColor: Utils.generateColors(Object.keys(categories).length) }] }; chartInstance = new Chart(ctx, { type: 'pie', data: chartData, options: { responsive: true } }); } return { renderChart }; })(); const Crawling = (() => { let isCrawlingFlag = Config.settings.isCrawling; let currentPage = parseInt(GM_getValue('currentPage', 1), 10); let totalPages = Config.settings.maxPages; let progress = parseFloat(GM_getValue('progress', 0)); let currentDepth = parseInt(GM_getValue('currentDepth', 0), 10); let activeRequests = 0; let requestQueue = []; let previousCrawls = JSON.parse(GM_getValue('previousCrawls', '[]')); let visitedUrls = new Set(JSON.parse(GM_getValue('visitedUrls', '[]'))); let data = JSON.parse(GM_getValue('data', '[]')); let navigationObserver = null; const searchEngineHandlers = { google: { extractResults: doc => { const results = []; doc.querySelectorAll('div.g, div[data-hveid]').forEach(item => { const name = item.querySelector('h3')?.innerText.trim() || 'No Title'; const url = item.querySelector('a')?.href || ''; const desc = item.querySelector('.aCOpRe, .VwiC3b')?.innerText.trim() || 'No Description'; if (Utils.isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: doc => doc.querySelector('a#pnnext')?.href || null, getContentContainer: () => '#search' }, bing: { extractResults: doc => { const results = []; doc.querySelectorAll('.b_algo').forEach(item => { const name = item.querySelector('h2')?.innerText.trim() || 'No Title'; const url = item.querySelector('h2 a')?.href || ''; const desc = item.querySelector('.b_caption p')?.innerText.trim() || 'No Description'; if (Utils.isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: doc => doc.querySelector('a.sb_pagN')?.href || null, getContentContainer: () => '#b_content' }, yahoo: { extractResults: doc => { const results = []; doc.querySelectorAll('.dd.algo').forEach(item => { const name = item.querySelector('h3')?.innerText.trim() || 'No Title'; const url = item.querySelector('a')?.href || ''; const desc = item.querySelector('.compText a')?.innerText.trim() || 'No Description'; if (Utils.isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: doc => doc.querySelector('a.next')?.href || null, getContentContainer: () => '#web' }, duckduckgo: { extractResults: doc => { const results = []; doc.querySelectorAll('.result').forEach(item => { const name = item.querySelector('h2')?.innerText.trim() || 'No Title'; const url = item.querySelector('a.result__a')?.href || ''; const desc = item.querySelector('.result__snippet')?.innerText.trim() || 'No Description'; if (Utils.isValidURL(url)) results.push({ name, url, description: desc }); }); return results; }, getNextPageUrl: doc => doc.querySelector('a.result--more__btn')?.href || null, getContentContainer: () => '#links' }, // 추가적인 검색 엔진은 여기서 정의할 수 있습니다. }; function identifySearchEngine() { const host = window.location.hostname; if (host.includes('google.com')) return 'google'; if (host.includes('bing.com') && !window.location.pathname.includes('/images')) return 'bing'; if (host.includes('yahoo.com')) return 'yahoo'; if (host.includes('duckduckgo.com')) return 'duckduckgo'; // 추가적인 검색 엔진 식별 로직을 여기에 추가하세요. return null; } async function startCrawling() { if (isCrawlingFlag) { Logger.log('이미 크롤링이 진행 중입니다.', 'warning'); return; } if (!Object.values(Config.settings.searchEngines).some(v => v)) { Logger.log('사용할 검색 엔진이 하나도 선택되지 않았습니다.', 'error'); return; } isCrawlingFlag = true; Config.saveSettings({ isCrawling: isCrawlingFlag }); Logger.log('크롤링을 시작합니다.', 'info'); UI.updateLogs(); if (!previousCrawls.some(crawl => crawl.status === '진행 중')) { previousCrawls.push({ date: new Date().toLocaleString(), count: 0, status: '진행 중' }); GM_setValue('previousCrawls', JSON.stringify(previousCrawls)); UI.updatePreviousCrawlsUI(); } setButtonStates(); await crawlPage(currentDepth); } function stopCrawling() { if (!isCrawlingFlag) { Logger.log('크롤링이 진행 중이지 않습니다.', 'warning'); return; } isCrawlingFlag = false; Config.saveSettings({ isCrawling: isCrawlingFlag }); Logger.log('크롤링이 중지되었습니다.', 'warning'); finalizeCrawl('중지됨'); UI.updateLogs(); setButtonStates(); } function setButtonStates() { $('#start-crawling').prop('disabled', isCrawlingFlag); $('#stop-crawling').prop('disabled', !isCrawlingFlag); } async function crawlPage(depth) { try { while (isCrawlingFlag && currentPage <= totalPages && depth <= Config.settings.maxDepth) { Logger.log(`페이지 ${currentPage} 크롤링 시작. (깊이: ${depth})`, 'info'); if (Config.settings.respectRobots && !(await Robots.checkRobotsTxt(window.location.href))) { Logger.log(`robots.txt에 의해 크롤링이 금지된 페이지: ${window.location.href}`, 'warning'); break; } const searchEngine = identifySearchEngine(); if (!searchEngine || !searchEngineHandlers[searchEngine]) { Logger.log(`지원되지 않는 검색 엔진: ${window.location.hostname}`, 'error'); break; } const doc = document; const handler = searchEngineHandlers[searchEngine]; const results = handler.extractResults(doc); if (!results) { Logger.log('검색 결과를 찾을 수 없습니다.', 'error'); break; } results.forEach(result => { if (!isCrawlingFlag) return; const { name, url, description } = result; if (shouldSkip(url, description)) return; requestQueue.push({ url, depth }); if (activeRequests >= Config.settings.maxConcurrentRequests) return; }); if (previousCrawls.length && previousCrawls[previousCrawls.length -1].status === '진행 중') { previousCrawls[previousCrawls.length -1].count += results.length; GM_setValue('previousCrawls', JSON.stringify(previousCrawls)); UI.updatePreviousCrawlsUI(); } await processQueue(); progress = (currentPage / totalPages) * 100; Config.saveSettings({ progress }); UI.updateLogs(); if (currentPage < totalPages && isCrawlingFlag) { const nextUrl = handler.getNextPageUrl(doc); if (nextUrl) { currentPage++; Config.saveSettings({ currentPage }); depth++; Config.saveSettings({ currentDepth: depth }); Logger.log(`다음 페이지로 이동: ${currentPage}`, 'info'); await Utils.delay(Config.settings.delay + Math.random() * Config.settings.delayRandom); const nextPageContent = await Fetch.fetchPageContent(nextUrl); if (nextPageContent) { // Parse the fetched content without altering the current page's DOM const parser = new DOMParser(); const fetchedDoc = parser.parseFromString(nextPageContent, 'text/html'); // Temporarily replace document's body with fetched content for extraction const originalBody = document.body.innerHTML; document.body.innerHTML = fetchedDoc.body.innerHTML; continue; } else { Logger.log('다음 페이지 내용을 가져올 수 없습니다.', 'error'); break; } } else { Logger.log('다음 페이지 링크를 찾을 수 없습니다.', 'error'); break; } } else { break; } } if (isCrawlingFlag && currentPage > totalPages && activeRequests === 0 && requestQueue.length === 0) { Logger.log('크롤링이 완료되었습니다.', 'success'); finalizeCrawl('완료됨'); isCrawlingFlag = false; Config.saveSettings({ isCrawling: isCrawlingFlag }); setButtonStates(); Config.saveSettings({ progress: 100 }); UI.updateLogs(); } } catch (error) { Logger.log(`크롤링 중 오류 발생: ${error.message}`, 'error'); finalizeCrawl('오류 발생'); } } function shouldSkip(url, desc) { if (visitedUrls.has(url) || !Utils.isValidURL(url)) return true; if (Config.settings.whitelist.length && !Config.settings.whitelist.some(d => url.includes(d))) return true; if (Config.settings.blacklist.some(d => url.includes(d))) return true; if (Config.settings.includeKeywords.length && !Config.settings.includeKeywords.some(k => url.includes(k) || desc.includes(k))) return true; if (Config.settings.excludeKeywords.some(k => url.includes(k) || desc.includes(k))) return true; if (Config.settings.regexInclude.length && !Config.settings.regexInclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(desc))) return true; if (Config.settings.regexExclude.length && Config.settings.regexExclude.some(r => new RegExp(r).test(url) || new RegExp(r).test(desc))) return true; return false; } async function processQueue() { while (requestQueue.length && activeRequests < Config.settings.maxConcurrentRequests && isCrawlingFlag) { const { url, depth } = requestQueue.shift(); activeRequests++; handleRequest(url, depth).finally(() => { activeRequests--; processQueue(); }); } } async function handleRequest(url, depth) { if (!isCrawlingFlag || visitedUrls.has(url)) return; try { const response = await Fetch.fetchWithRetry(url, Config.settings.retryAttempts, Config.settings.retryDelay); const contentType = Fetch.getContentType(response); if (!Fetch.isContentTypeAllowed(contentType, Config.settings.contentTypes)) { Logger.log(`허용되지 않은 콘텐츠 유형(${contentType})의 URL: ${url}`, 'warning'); return; } const parser = new DOMParser(); const doc = parser.parseFromString(response.responseText, 'text/html'); const text = doc.body?.innerText || ''; if (Config.settings.enableCaptchas && Robots.detectCaptcha(doc)) { await Robots.handleCaptcha(url); return; } const wc = text.split(/\s+/).length; if (wc < Config.settings.minWordCount || wc > Config.settings.maxWordCount) { Logger.log(`단어 수 조건에 맞지 않는 URL: ${url} (${wc} 단어)`, 'warning'); return; } const quality = Evaluate.evaluateContentQuality(text); const lang = doc.documentElement.lang || 'unknown'; const category = Classifier.classifyKeywords(text); const meta = Extractor.extractMetaTags(doc); const headings = Extractor.extractHeadings(doc); if (category !== '기타' || quality > 100) { const entry = { url, quality, language: lang, wordCount: wc, category }; Config.settings.dataFields.name && (entry.name = Extractor.extractName(doc) || 'No Title'); Config.settings.dataFields.description && (entry.description = Extractor.extractDescription(doc) || 'No Description'); Config.settings.dataFields.html && (entry.html = response.responseText); Config.settings.dataFields.metaTags && (entry.metaTags = meta); Config.settings.dataFields.headings && (entry.headings = headings); data.push(entry); GM_setValue('data', JSON.stringify(data)); UI.updateDataTable(); Logger.log(`데이터 수집 완료: ${url}`, 'success'); Config.settings.dataVisualization && DataVisualization.renderChart(); } if (depth < Config.settings.maxDepth) { Extractor.extractLinks(doc).forEach(link => { if (!visitedUrls.has(link) && Utils.isValidURL(link)) requestQueue.push({ url: link, depth: depth + 1 }); }); } } catch (error) { Logger.log(`데이터 수집 실패: ${url} - ${error.message}`, 'error'); } visitedUrls.add(url); GM_setValue('visitedUrls', JSON.stringify([...visitedUrls])); } function finalizeCrawl(status) { if (previousCrawls.length) { previousCrawls[previousCrawls.length -1].status = status; GM_setValue('previousCrawls', JSON.stringify(previousCrawls)); UI.updatePreviousCrawlsUI(); } if (navigationObserver) { navigationObserver.disconnect(); navigationObserver = null; } } function updateCrawlSettings() { // 필요한 경우 크롤링 설정 업데이트 로직을 여기에 추가하세요. } return { startCrawling, stopCrawling, isCrawling: () => isCrawlingFlag, updateCrawlSettings }; })(); const Fetch = (() => { async function fetchWithRetry(url, attempts, delayMs) { for (let i = 1; i <= attempts; i++) { try { return await GM_xmlhttpRequestPromise(url); } catch (e) { Logger.log(`요청 실패 (${i}/${attempts}): ${url} - ${e.message}`, 'warning'); if (i < attempts) await Utils.delay(delayMs); } } throw new Error('모든 재시도 시도가 실패했습니다.'); } function GM_xmlhttpRequestPromise(url) { return new Promise((resolve, reject) => { let reqUrl = Config.settings.proxy ? `${Config.settings.proxy}?url=${encodeURIComponent(url)}` : url; GM_xmlhttpRequest({ method: "GET", url: reqUrl, headers: { 'User-Agent': getCurrentUserAgent(), 'Accept': '*/*' }, timeout: Config.settings.timeout, onload: res => (res.status >= 200 && res.status < 300) ? resolve(res) : reject(new Error(`HTTP ${res.status}`)), onerror: () => reject(new Error('Request failed')), ontimeout: () => reject(new Error('Request timed out')) }); }); } function getCurrentUserAgent() { const index = Config.settings.userAgentIndex % Config.settings.userAgents.length; return Config.settings.userAgents[index]; } function getContentType(res) { return res.responseHeaders.split('\n').find(h => h.toLowerCase().startsWith('content-type'))?.split(':')[1].trim().split(';')[0] || ''; } function isContentTypeAllowed(ct, contentTypes) { if (contentTypes.include.length && !contentTypes.include.includes(ct)) return false; if (contentTypes.exclude.includes(ct)) return false; return true; } async function fetchPageContent(url) { try { const response = await fetchWithRetry(url, Config.settings.retryAttempts, Config.settings.retryDelay); return response.responseText; } catch (error) { Logger.log(`페이지 내용 가져오기 실패: ${url} - ${error.message}`, 'error'); return null; } } return { fetchWithRetry, getCurrentUserAgent, getContentType, isContentTypeAllowed, fetchPageContent }; })(); const Robots = (() => { async function checkRobotsTxt(url) { try { const parsed = new URL(url); const robotsUrl = `${parsed.origin}/robots.txt`; const res = await Fetch.fetchWithRetry(robotsUrl, 2, 2000); return new RobotsParser(res.responseText).isAllowed(url, Fetch.getCurrentUserAgent()); } catch { Logger.log('robots.txt 확인 실패, 기본 허용.', 'warning'); return true; } } function detectCaptcha(doc) { return !!doc.querySelector('iframe[src*="captcha"], form[action*="captcha"]'); } async function handleCaptcha(url) { Logger.log(`CAPTCHA 감지됨: ${url}`, 'warning'); await Utils.delay(5000); Logger.log(`CAPTCHA 처리 완료: ${url}`, 'info'); } class RobotsParser { constructor(text) { this.rules = []; this.parse(text); } parse(text) { let agents = []; text.split('\n').forEach(line => { const [key, ...val] = line.split(':').map(s => s.trim()); if (!key) return; if (key.toLowerCase() === 'user-agent') agents = [val.join(':').toLowerCase()]; else if (['disallow','allow'].includes(key.toLowerCase())) this.rules.push({ agents, type: key.toLowerCase(), path: val.join(':') }); }); } isAllowed(url, ua) { const parsedUrl = new URL(url); const path = parsedUrl.pathname; let allowed = true; this.rules.forEach(rule => { if (rule.agents.includes('*') || rule.agents.includes(ua.toLowerCase())) { if (rule.type === 'disallow' && path.startsWith(rule.path)) allowed = false; if (rule.type === 'allow' && path.startsWith(rule.path)) allowed = true; } }); return allowed; } } return { checkRobotsTxt, detectCaptcha, handleCaptcha }; })(); const Extractor = (() => { function extractMetaTags(doc) { const metas = {}; doc.querySelectorAll('meta[name], meta[property]').forEach(meta => { const name = meta.getAttribute('name') || meta.getAttribute('property'); const content = meta.getAttribute('content'); if (name && content) metas[name.toLowerCase()] = content; }); return metas; } function extractHeadings(doc) { const hs = []; for (let i = 1; i <= 6; i++) doc.querySelectorAll(`h${i}`).forEach(h => hs.push(h.innerText.trim())); return hs; } function extractName(doc) { return doc.querySelector('title')?.innerText || 'No Title'; } function extractDescription(doc) { return doc.querySelector('meta[name="description"]')?.getAttribute('content') || 'No Description'; } function extractLinks(doc) { return Array.from(doc.querySelectorAll('a[href]')).map(a => a.href).filter(Utils.isValidURL); } return { extractMetaTags, extractHeadings, extractName, extractDescription, extractLinks }; })(); const Classifier = (() => { function classifyKeywords(text) { const kws = combineKeywords(); for (const [cat, keywords] of Object.entries(kws)) { if (keywords.some(kw => text.includes(kw))) return cat; } return '기타'; } function combineKeywords() { const standard = { '교육': ['학교', '교육', '학생', '교수', '강의'], '기술': ['기술', '소프트웨어', '하드웨어', '프로그래밍', '개발'], '뉴스': ['뉴스', '속보', '기사', '보도'], '엔터테인먼트': ['영화', '음악', '연예', '게임', '드라마'], '스포츠': ['축구', '야구', '농구', '경기', '토너먼트'], '비즈니스': ['비즈니스', '경제', '주식', '투자', '시장'], '건강': ['건강', '의료', '헬스', '운동', '영양'], '과학': ['과학', '연구', '실험', '발견', '이론'], '문학': ['소설', '시', '문학', '작가', '책'], '여행': ['여행', '관광', '숙박', '비행기', '여권'] }; const custom = Config.settings.customKeywords || {}; for (const [cat, kws] of Object.entries(custom)) { standard[cat] = standard[cat] ? [...new Set([...standard[cat], ...kws])] : kws; } return standard; } return { classifyKeywords }; })(); const Evaluate = (() => { function evaluateContentQuality(text) { return text.split(/\s+/).length; } return { evaluateContentQuality }; })(); const Scheduler = (() => { let scheduleInterval = null; function scheduleCrawling() { clearInterval(scheduleInterval); const now = new Date(); const [startHour, startMinute] = Config.settings.schedule.startTime.split(':').map(Number); const startTime = new Date(now.getFullYear(), now.getMonth(), now.getDate(), startHour, startMinute, 0); if (startTime < now) startTime.setDate(startTime.getDate() + 1); const initialDelay = startTime - now; setTimeout(() => { if (Config.settings.schedule.specificDays.length === 0 || Config.settings.schedule.specificDays.includes(startTime.getDay())) { Crawling.startCrawling(); } scheduleInterval = setInterval(() => { const currentDay = new Date().getDay(); if (Config.settings.schedule.specificDays.length === 0 || Config.settings.schedule.specificDays.includes(currentDay)) { Crawling.startCrawling(); } }, Config.settings.schedule.interval); }, initialDelay); Logger.log(`크롤링 스케줄이 설정되었습니다. 시작 시간: ${startTime.toLocaleString()}, 간격: ${Config.settings.schedule.interval / 60000}분`, 'info'); } return { scheduleCrawling }; })(); const Testing = (() => { function runTests() { $('#test-modal').show(); $('#mocha').html(''); $('head').append('<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/mocha/10.2.0/mocha.min.css" />'); mocha.setup('bdd'); const assert = chai.assert; describe('Utils Module', function() { it('should escape HTML correctly', function() { const input = '<div>Hello & "World"!</div>'; const expected = '<div>Hello & "World"!</div>'; assert.equal(Utils.escape(input, 'html'), expected); }); it('should capitalize strings correctly', function() { const input = 'test'; const expected = 'Test'; assert.equal(Utils.capitalize(input), expected); }); it('should validate URLs correctly', function() { assert.isTrue(Utils.isValidURL('https://www.example.com')); assert.isFalse(Utils.isValidURL('htp:/invalid-url')); }); }); describe('Config Module', function() { it('should save and load settings correctly', function() { const originalDelay = Config.settings.delay; Config.saveSettings({ delay: 3000 }); assert.equal(Config.settings.delay, 3000); Config.saveSettings({ delay: originalDelay }); }); it('should add and remove templates correctly', function() { const templateName = 'TestTemplate'; Config.saveTemplate(templateName); assert.property(Config.settings.templates, templateName); Config.loadTemplate(templateName); Config.settings.templates[templateName] = undefined; delete Config.settings.templates[templateName]; assert.notProperty(Config.settings.templates, templateName); }); }); describe('Logger Module', function() { it('should log messages correctly', function() { Logger.log('Test log message', 'info'); const logs = Logger.getLogs(); assert.isAtLeast(logs.length, 1); assert.equal(logs[logs.length -1].message, 'Test log message'); }); it('should clear logs correctly', function() { Logger.clearLogs(); assert.equal(Logger.getLogs().length, 0); }); }); mocha.run(); } return { runTests }; })(); function init() { UI.createUI(); UI.updateUI(); if (Config.settings.dataVisualization) DataVisualization.renderChart(); if (Config.settings.schedule.enabled) Scheduler.scheduleCrawling(); if (Config.settings.isCrawling) { Logger.log('크롤링 재개됨.', 'info'); Crawling.startCrawling(); } else { Logger.log('크롤러 초기화됨.', 'info'); } } window.addEventListener('load', init);})(); UI 개선 ㄱㄱ 코드만 쓰셈 ㄱㄱ
- Assistant
const UI = (() => { function createUI() { GM_addStyle(` /* Highlight.js default style */ .hljs { display: block; overflow-x: auto; padding: 1em; background: #f0f0f0; border-radius: 8px; font-size: 14px; } /* Dark mode adjustments for Highlight.js */ #crawler-container.dark .hljs { background: #2b2b2b; color: #f1f1f1; } /* Existing styles */ #crawler-container { position: fixed; top: 20px; right: 20px; width: 450px; max-width: 90%; height: 650px; background: var(--bg-color, #fff); border: 2px solid #ccc; z-index: 10000; font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; box-shadow: 0 4px 16px rgba(0,0,0,0.3); overflow: hidden; resize: both; border-radius: 10px; min-width: 320px; min-height: 300px; } #crawler-container.dark { --bg-color: #2b2b2b; color: #f1f1f1; border-color: #555; } #crawler-header { display: flex; justify-content: space-between; align-items: center; background: var(--header-bg, #f1f1f1); border-bottom: 2px solid #ccc; padding: 12px 16px; cursor: move; } #crawler-container.dark #crawler-header { --header-bg: #3c3c3c; border-color: #555; } #crawler-minimize, #dark-mode-toggle { background: none; border: none; font-size: 20px; cursor: pointer; color: inherit; margin-left: 8px; } #crawler-tabs { display: flex; flex-wrap: wrap; gap: 8px; overflow-x: auto; } .crawler-tab { padding: 10px 14px; cursor: pointer; font-weight: bold; border-radius: 6px; background: #e0e0e0; white-space: nowrap; flex: 0 0 auto; transition: background 0.3s; } .crawler-tab.active { background: var(--active-tab-bg, #bdbdbd); box-shadow: inset 0 -3px 0 #1976d2; color: #1976d2; } #crawler-container.dark .crawler-tab { background: #4a4a4a; } #crawler-container.dark .crawler-tab.active { background: #616161; color: #90caf9; box-shadow: inset 0 -3px 0 #90caf9; } #crawler-content { padding: 16px; overflow-y: auto; height: calc(100% - 60px); background: var(--content-bg, #fff); } #crawler-container.dark #crawler-content { --content-bg: #3c3c3c; } table { width: 100%; border-collapse: collapse; table-layout: fixed; word-wrap: break-word; } th, td { padding: 10px; text-align: left; border-bottom: 1px solid #ddd; } th { background-color: var(--th-bg, #f2f2f2); cursor: pointer; position: sticky; top: 0; z-index: 1; } #crawler-container.dark th { background-color: #444; } tr:hover { background-color: #f5f5f5; } #crawler-container.dark tr:hover { background-color: #555; } .progress-bar { width: 100%; background-color: #f3f3f3; border: 1px solid #ccc; border-radius: 6px; overflow: hidden; margin-bottom: 12px; } .progress-bar-fill { height: 24px; background-color: #4caf50; width: 0%; transition: width 0.3s; } .modal { display: none; position: fixed; z-index: 10001; left: 0; top: 0; width: 100%; height: 100%; overflow: auto; background-color: rgba(0,0,0,0.5); } .modal-content { background-color: var(--modal-bg, #fff); margin: 10% auto; padding: 20px; border: 2px solid #888; width: 85%; max-width: 700px; max-height: 80%; overflow-y: auto; border-radius: 10px; position: relative; } #crawler-container.dark .modal-content { --modal-bg: #3c3c3c; color: #f1f1f1; border-color: #555; } .close { color: #aaa; position: absolute; top: 10px; right: 20px; font-size: 30px; font-weight: bold; cursor: pointer; } .close:hover, .close:focus { color: #000; text-decoration: none; cursor: pointer; } .control-buttons { margin: 16px 0; display: flex; flex-wrap: wrap; gap: 12px; } .control-buttons button { padding: 10px 16px; cursor: pointer; border: none; background-color: #4caf50; color: white; border-radius: 6px; font-size: 14px; flex: 1 1 auto; min-width: 100px; transition: background-color 0.3s; } .control-buttons button:hover { background-color: #45a049; } .control-buttons button:disabled { background-color: #a5d6a7; cursor: not-allowed; } .status-text { margin-top: 12px; font-weight: bold; font-size: 16px; text-align: center; } .settings-section { margin-bottom: 18px; } .settings-section label { display: block; margin-bottom: 6px; font-weight: bold; font-size: 14px; } .settings-section input[type="text"], .settings-section input[type="number"], .settings-section textarea, .settings-section select { width: 100%; padding: 8px; box-sizing: border-box; border: 1px solid #ccc; border-radius: 6px; font-size: 14px; transition: border-color 0.3s; } .settings-section input[type="text"]:focus, .settings-section input[type="number"]:focus, .settings-section textarea:focus, .settings-section select:focus { border-color: #1976d2; } .settings-section.dark .settings-section input, .settings-section.dark .settings-section select, .settings-section.dark .settings-section textarea { background-color: #555; color: #f1f1f1; border-color: #777; } .checkbox-group, .radio-group { display: flex; flex-wrap: wrap; gap: 12px; } .checkbox-group label, .radio-group label { font-size: 14px; display: flex; align-items: center; gap: 6px; } .tooltip { position: relative; display: inline-block; cursor: help; font-size: 14px; margin-left: 4px; } .tooltip .tooltiptext { visibility: hidden; width: 220px; background-color: #555; color: #fff; text-align: center; border-radius: 6px; padding: 6px 8px; position: absolute; z-index: 1; bottom: 125%; left: 50%; margin-left: -110px; opacity: 0; transition: opacity 0.3s; } .tooltip:hover .tooltiptext { visibility: visible; opacity: 1; } #crawler-container::-webkit-resizer { background: #f1f1f1; border: none; } #data-visualization { margin-top: 24px; } #data-chart { width: 100%; height: 350px; } #modal-html-content { white-space: pre-wrap; word-wrap: break-word; background-color: #f9f9f9; padding: 12px; border-radius: 6px; overflow-x: auto; } #crawler-container.dark #modal-html-content { background-color: #3c3c3c; color: #f1f1f1; } #previous-crawls table, #previous-crawls th, #previous-crawls td { width: 100%; padding: 8px; border: 1px solid #ddd; border-radius: 4px; } #reset-data { background-color: #f44336; flex: 1 1 auto; } .spinner { border: 4px solid #f3f3f3; border-top: 4px solid #3498db; border-radius: 50%; width: 24px; height: 24px; animation: spin 2s linear infinite; display: inline-block; vertical-align: middle; } @keyframes spin { 0% { transform: rotate(0deg); } 100% { transform: rotate(360deg); } } .error-message { color: #f44336; font-weight: bold; margin-top: 12px; text-align: center; } .success-message { color: #4caf50; font-weight: bold; margin-top: 12px; text-align: center; } .dark-mode-toggle { background: none; border: none; cursor: pointer; font-size: 20px; color: inherit; } @media (max-width: 768px) { #crawler-container { width: 100%; height: 100%; top: 0; right: 0; border-radius: 0; } #crawler-header { flex-direction: column; align-items: flex-start; } #crawler-tabs { flex-direction: row; overflow-x: auto; } .crawler-tab { flex: 0 0 auto; } } /* Additional CSS: Full URL display with tooltip */ td a.full-url { display: block; word-break: break-all; position: relative; text-decoration: none; color: #1976d2; } td a.full-url:hover::after { content: attr(href); position: absolute; background: #333; color: #fff; padding: 6px 8px; border-radius: 6px; white-space: nowrap; z-index: 10002; top: 100%; left: 0; margin-top: 4px; font-size: 12px; pointer-events: none; opacity: 0.9; } `); const container = $(` <div id="crawler-container"> <div id="crawler-header"> <div id="crawler-tabs"> <div class="crawler-tab" data-tab="logs" title="View Logs">📜 로그</div> <div class="crawler-tab" data-tab="data" title="View Collected Data">📊 데이터</div> <div class="crawler-tab" data-tab="settings" title="Change Settings">⚙️ 설정</div> <div class="crawler-tab" data-tab="previous-crawls" title="Previous Crawling Records">📁 기록</div> <div class="crawler-tab" data-tab="templates" title="Settings Templates">📂 템플릿</div> <div class="crawler-tab" data-tab="tests" title="Run Tests">🧪 테스트</div> <button id="dark-mode-toggle" class="dark-mode-toggle" title="Toggle Theme">🌙</button> </div> <button id="crawler-minimize" title="Minimize Window">─</button> </div> <div id="crawler-content"> <div id="tab-logs"> <div class="progress-bar"> <div class="progress-bar-fill" style="width: 0%;"></div> </div> <div class="status-text">진행률: 0% (0/0 페이지)</div> <div class="control-buttons"> <button id="start-crawling" title="Start Crawling">▶ 시작</button> <button id="stop-crawling" title="Stop Crawling" disabled>■ 중지</button> ${Config.settings.exportFormats.csv ? `<button id="download-csv" title="Download CSV">📥 CSV</button>` : ''} ${Config.settings.exportFormats.json ? `<button id="download-json" title="Download JSON">📥 JSON</button>` : ''} ${Config.settings.exportFormats.xml ? `<button id="download-xml" title="Download XML">📥 XML</button>` : ''} ${Config.settings.exportFormats.xls ? `<button id="download-xls" title="Download XLS">📥 XLS</button>` : ''} <button id="reset-data" title="Reset All Crawling Data">🗑️ 초기화</button> </div> <div id="log-messages" style="height: 220px; overflow-y: auto; background: var(--log-bg, #f9f9f9); padding: 12px; border: 1px solid #ccc; border-radius: 6px;"> 로그 내용이 여기에 표시됩니다. </div> <div id="error-message" class="error-message" style="display: none;"></div> <div id="success-message" class="success-message" style="display: none;"></div> </div> <div id="tab-data" style="display: none;"> <input type="text" id="search-filter" placeholder="검색 필터..." style="width: 100%; padding: 8px; margin-bottom: 12px; border: 1px solid #ccc; border-radius: 6px;"> <table> <thead> <tr> ${Config.settings.dataFields.name ? '<th data-sort="name">사이트 이름</th>' : ''} ${Config.settings.dataFields.url ? '<th data-sort="url">주소</th>' : ''} ${Config.settings.dataFields.description ? '<th data-sort="description">설명</th>' : ''} ${Config.settings.dataFields.html ? '<th>📄 HTML 내용</th>' : ''} ${Config.settings.dataFields.category ? '<th data-sort="category">분류</th>' : ''} ${Config.settings.dataFields.quality ? '<th data-sort="quality">품질점수</th>' : ''} ${Config.settings.dataFields.language ? '<th data-sort="language">언어</th>' : ''} ${Config.settings.dataFields.wordCount ? '<th data-sort="wordCount">단어 수</th>' : ''} ${Config.settings.dataFields.metaTags ? '<th>메타 태그</th>' : ''} ${Config.settings.dataFields.headings ? '<th>헤딩</th>' : ''} </tr> </thead> <tbody id="data-table-body"></tbody> </table> ${Config.settings.dataVisualization ? ` <div id="data-visualization"> <h3>데이터 시각화</h3> <canvas id="data-chart"></canvas> </div> ` : ''} </div> <div id="tab-settings" style="display: none;"> <h3>설정</h3> ${createSettingsSections()} <div class="settings-section"> <button id="save-settings" style="margin-top: 10px; padding: 12px 24px; background-color: #1976d2; color: white; border-radius: 6px; font-size: 14px;">저장</button> <button id="reset-settings" style="margin-top: 10px; padding: 12px 24px; background-color: #d32f2f; color: white; border-radius: 6px; font-size: 14px;">초기화</button> </div> </div> <div id="tab-templates" style="display: none;"> <h3>설정 템플릿</h3> <div class="settings-section"> <label for="new-template-name">새 템플릿 이름:</label> <input type="text" id="new-template-name" placeholder="템플릿 이름을 입력하세요"> <button id="save-template" style="margin-top: 8px; padding: 10px 20px; background-color: #1976d2; color: white; border-radius: 6px; font-size: 14px;">📁 템플릿 저장</button> </div> <div class="settings-section"> <label for="template-select">기존 템플릿:</label> <select id="template-select" style="width: 100%; padding: 8px; margin-top: 10px; border: 1px solid #ccc; border-radius: 6px;"> <option value="" disabled selected>템플릿 선택</option> ${Object.keys(Config.settings.templates).map(t => `<option value="${t}">${t}</option>`).join('')} </select> <button id="load-template" style="margin-top: 12px; padding: 10px 20px; background-color: #388e3c; color: white; border-radius: 6px; font-size: 14px;">📂 템플릿 불러오기</button> <button id="delete-template" style="margin-top: 8px; padding: 10px 20px; background-color: #d32f2f; color: white; border-radius: 6px; font-size: 14px;">템플릿 삭제</button> </div> </div> <div id="tab-previous-crawls" style="display: none;"> <h3>이전 크롤링 기록</h3> <button id="clear-previous-crawls" style="padding: 10px 20px; background-color: #d32f2f; color: white; border-radius: 6px; font-size: 14px; margin-bottom: 16px;">기록 초기화</button> <table> <thead> <tr> <th>날짜</th> <th>크롤링 수</th> <th>상태</th> </tr> </thead> <tbody id="previous-crawls-body"></tbody> </table> </div> <div id="tab-tests" style="display: none;"> <button id="run-tests" style="padding: 12px 24px; background-color: #ffa000; color: white; border-radius: 6px; font-size: 14px;">테스트 실행</button> </div> <div id="html-modal" class="modal"> <div class="modal-content"> <span class="close">×</span> <pre><code id="modal-html-content" class="hljs language-html"></code></pre> </div> </div> <div id="test-modal" class="modal"> <div class="modal-content"> <span class="close">×</span> <div id="mocha"></div> </div> </div> </div> </div> `); $('body').append(container); bindEvents(); makeDraggable(); } function createSettingsSections() { const sections = [ { label: '크롤링 딜레이 (ms)', id: 'delay-input', type: 'number', value: Config.settings.delay, min: 500, step: 500, tooltip: '기본 딜레이와 딜레이 랜덤 값을 조합하여 요청 사이의 딜레이를 설정합니다.' }, { label: '딜레이 랜덤 범위 (ms)', id: 'delay-random-input', type: 'number', value: Config.settings.delayRandom, min: 0, step: 500, tooltip: '딜레이에 무작위 요소를 추가하여 보다 자연스러운 요청을 합니다.' }, { label: '최대 크롤링 페이지 수', id: 'max-pages-input', type: 'number', value: Config.settings.maxPages, min: 1, step: 1, tooltip: '크롤링할 최대 페이지 수를 설정하세요.' }, { label: '사용자 에이전트(User-Agent)', id: 'user-agent-select', type: 'select-multiple', options: Config.settings.userAgents.map((ua, idx) => ({ value: idx, text: ua, selected: idx === Config.settings.userAgentIndex })), buttons: [ { id: 'add-user-agent', text: '추가', bgColor: '#1976d2' }, { id: 'remove-user-agent', text: '제거', bgColor: '#d32f2f' } ], tooltip: '크롤링에 사용할 사용자 에이전트를 선택하거나 추가할 수 있습니다.' }, { label: '콘텐츠 유형 포함', id: 'content-include-input', type: 'text', value: Config.settings.contentTypes.include.join(', '), tooltip: '크롤링할 콘텐츠 유형을 쉼표로 구분하여 입력하세요.' }, { label: '콘텐츠 유형 제외', id: 'content-exclude-input', type: 'text', value: Config.settings.contentTypes.exclude.join(', '), tooltip: '크롤링에서 제외할 콘텐츠 유형을 쉼표로 구분하여 입력하세요.' }, { label: '크롤링 타임아웃 (ms)', id: 'timeout-input', type: 'number', value: Config.settings.timeout, min: 1000, step: 1000, tooltip: '요청이 완료되기까지 기다릴 최대 시간을 밀리초 단위로 설정합니다.' }, { label: '포함할 키워드', id: 'include-keywords-input', type: 'textarea', value: Config.settings.includeKeywords.join('\n'), tooltip: '크롤링할 페이지에 반드시 포함되어야 하는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.' }, { label: '제외할 키워드', id: 'exclude-keywords-input', type: 'textarea', value: Config.settings.excludeKeywords.join('\n'), tooltip: '크롤링할 페이지에 포함되면 안 되는 키워드를 입력하세요. 각 키워드는 줄바꿈으로 구분됩니다.' }, { label: '포함할 키워드 (정규 표현식)', id: 'regex-include-input', type: 'textarea', value: Config.settings.regexInclude.join('\n'), tooltip: '포함할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.' }, { label: '제외할 키워드 (정규 표현식)', id: 'regex-exclude-input', type: 'textarea', value: Config.settings.regexExclude.join('\n'), tooltip: '제외할 키워드의 정규 표현식을 입력하세요. 각 패턴은 줄바꿈으로 구분됩니다.' }, { label: '언어 필터링', id: 'language-select', type: 'select-multiple', options: [ { value: 'en', text: '영어 (en)' }, { value: 'ko', text: '한국어 (ko)' }, { value: 'zh', text: '중국어 (zh)' }, { value: 'ja', text: '일본어 (ja)' }, { value: 'ru', text: '러시아어 (ru)' }, { value: 'es', text: '스페인어 (es)' }, { value: 'fr', text: '프랑스어 (fr)' }, { value: 'de', text: '독일어 (de)' }, { value: 'it', text: '이탈리아어 (it)' }, { value: 'pt', text: '포르투갈어 (pt)' }, { value: 'ar', text: '아랍어 (ar)' }, { value: 'hi', text: '힌디어 (hi)' }, { value: 'bn', text: '벵골어 (bn)' }, { value: 'pa', text: '펀자브어 (pa)' }, { value: 'nl', text: '네덜란드어 (nl)' }, { value: 'sv', text: '스웨덴어 (sv)' }, { value: 'tr', text: '터키어 (tr)' }, { value: 'vi', text: '베트남어 (vi)' }, { value: 'pl', text: '폴란드어 (pl)' }, { value: 'th', text: '태국어 (th)' } ].map(lang => ({ value: lang.value, text: lang.text, selected: Config.settings.language.includes(lang.value) })) }, { label: '최소 단어 수', id: 'min-word-count-input', type: 'number', value: Config.settings.minWordCount, min: 10, step: 10, tooltip: '수집할 페이지의 최소 단어 수를 설정하세요.' }, { label: '최대 단어 수', id: 'max-word-count-input', type: 'number', value: Config.settings.maxWordCount, min: 100, step: 100, tooltip: '수집할 페이지의 최대 단어 수를 설정하세요.' }, { label: '최대 크롤링 깊이', id: 'max-depth-input', type: 'number', value: Config.settings.maxDepth, min: 1, step: 1, tooltip: '링크를 추적할 최대 깊이를 설정하세요.' }, { label: '프록시 서버', id: 'proxy-input', type: 'text', value: Config.settings.proxy, placeholder: 'http://proxyserver:port', tooltip: '프록시 서버 주소를 입력하세요. 예: http://proxyserver:port' }, { label: '수집할 데이터 필드', id: '', type: 'checkbox-group', options: Object.keys(Config.settings.dataFields).map(field => ({ id: `field-${field}`, text: Utils.capitalize(getFieldName(field)), checked: Config.settings.dataFields[field] })) }, { label: '재시도 횟수', id: 'retry-attempts-input', type: 'number', value: Config.settings.retryAttempts, min: 0, step: 1, tooltip: '요청 실패 시 재시도할 횟수를 설정하세요.' }, { label: '재시도 딜레이 (ms)', id: 'retry-delay-input', type: 'number', value: Config.settings.retryDelay, min: 1000, step: 500, tooltip: '재시도 시 대기할 시간을 밀리초 단위로 설정하세요.' }, { label: '크롤링 스케줄링', id: 'schedule-enabled', type: 'checkbox', checked: Config.settings.schedule.enabled, tooltip: '크롤링 스케줄링을 활성화합니다.' }, { label: '시작 시간 (HH:MM)', id: 'start-time-input', type: 'time', value: Config.settings.schedule.startTime, display: Config.settings.schedule.enabled, tooltip: '크롤링을 시작할 시간을 설정하세요.' }, { label: '크롤링 간격 (분)', id: 'interval-input', type: 'number', value: Config.settings.schedule.interval / 60000, min: 1, step: 1, display: Config.settings.schedule.enabled, tooltip: '크롤링 작업 간의 간격을 분 단위로 설정하세요.' }, { label: '특정 요일', id: 'specific-days-input', type: 'select-multiple', options: [ { value: '0', text: '일요일' }, { value: '1', text: '월요일' }, { value: '2', text: '화요일' }, { value: '3', text: '수요일' }, { value: '4', text: '목요일' }, { value: '5', text: '금요일' }, { value: '6', text: '토요일' } ].map(day => ({ value: day.value, text: day.text, selected: Config.settings.schedule.specificDays.includes(parseInt(day.value)) })), display: Config.settings.schedule.enabled, tooltip: '크롤링을 수행할 특정 요일을 선택하세요. (복수 선택 가능)' }, { label: '크롤링할 검색 엔진 선택', id: '', type: 'checkbox-group', options: Object.keys(Config.settings.searchEngines).map(engine => ({ id: `engine-${engine}`, text: Utils.capitalize(engine), checked: Config.settings.searchEngines[engine] })) }, { label: '블랙리스트 도메인', id: 'blacklist-input', type: 'textarea', value: Config.settings.blacklist.join('\n'), tooltip: '크롤링에서 제외할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.' }, { label: '화이트리스트 도메인', id: 'whitelist-input', type: 'textarea', value: Config.settings.whitelist.join('\n'), tooltip: '크롤링을 허용할 도메인을 입력하세요. 각 도메인은 줄바꿈으로 구분됩니다.' }, { label: 'robots.txt 준수', id: 'respect-robots-checkbox', type: 'checkbox', checked: Config.settings.respectRobots, tooltip: '대상 사이트의 robots.txt 규칙을 준수하여 크롤링을 진행합니다.' }, { label: '동시 요청 최대 수', id: 'max-concurrent-requests-input', type: 'number', value: Config.settings.maxConcurrentRequests, min: 1, step: 1, tooltip: '한 번에 요청할 수 있는 최대 동시 요청 수를 설정하세요.' }, { label: '데이터 시각화 활성화', id: 'data-visualization-checkbox', type: 'checkbox', checked: Config.settings.dataVisualization, tooltip: '수집된 데이터를 시각화하여 그래프로 표시합니다.' }, { label: 'CAPTCHA 처리 활성화', id: 'enable-captchas-checkbox', type: 'checkbox', checked: Config.settings.enableCaptchas, tooltip: 'CAPTCHA가 감지되었을 때 자동 처리를 활성화합니다. (제한적)' }, { label: '데이터 내보내기 형식', id: '', type: 'checkbox-group', options: Object.keys(Config.settings.exportFormats).map(format => ({ id: `export-${format}`, text: format.toUpperCase(), checked: Config.settings.exportFormats[format] })) }, { label: '고급 로깅 활성화', id: 'advanced-logging-checkbox', type: 'checkbox', checked: Config.settings.advancedLogging, tooltip: '추가적인 로깅 정보를 활성화합니다.' }, { label: '사용자 지정 키워드 분류', id: 'custom-keywords-input', type: 'textarea', value: Utils.formatCustomKeywords(Config.settings.customKeywords), tooltip: '사용자 지정 키워드를 추가하여 데이터 분류를 향상시킵니다. 형식: 카테고리:키워드1,키워드2,...' } ]; return sections.map(sec => { if (sec.type === 'checkbox-group') { return ` <div class="settings-section"> <label>${sec.label}:</label> <div class="checkbox-group"> ${sec.options.map(opt => `<label><input type="checkbox" id="${opt.id}" ${opt.checked ? 'checked' : ''}> ${opt.text}</label>`).join('')} </div> </div> `; } if (sec.type === 'select-multiple') { return ` <div class="settings-section"> <label for="${sec.id}">${sec.label}: <span class="tooltip">ℹ️ <span class="tooltiptext">${sec.tooltip}</span> </span> </label> <select id="${sec.id}" multiple size="${sec.options.length || 5}"> ${sec.options.map(opt => `<option value="${opt.value}" ${opt.selected ? 'selected' : ''}>${opt.text}</option>`).join('')} </select> </div> `; } if (sec.type === 'checkbox') { return ` <div class="settings-section"> <label for="${sec.id}">${sec.label}: <span class="tooltip">ℹ️ <span class="tooltiptext">${sec.tooltip}</span> </span> </label> <input type="checkbox" id="${sec.id}" ${sec.checked ? 'checked' : ''}> </div> `; } if (sec.type === 'textarea') { return ` <div class="settings-section"> <label for="${sec.id}">${sec.label}: <span class="tooltip">ℹ️ <span class="tooltiptext">${sec.tooltip}</span> </span> </label> <textarea id="${sec.id}" rows="4">${sec.value}</textarea> </div> `; } if (sec.type === 'select') { return ` <div class="settings-section"> <label for="${sec.id}">${sec.label}:</label> <select id="${sec.id}" ${sec.multiple ? 'multiple' : ''}> ${sec.options.map(opt => `<option value="${opt.value}" ${opt.selected ? 'selected' : ''}>${opt.text}</option>`).join('')} </select> </div> `; } return ` <div class="settings-section"> <label for="${sec.id}">${sec.label}: <span class="tooltip">ℹ️ <span class="tooltiptext">${sec.tooltip}</span> </span> </label> <input type="${sec.type}" id="${sec.id}" value="${sec.value}" ${sec.min ? `min="${sec.min}"` : ''} ${sec.step ? `step="${sec.step}"` : ''} ${sec.placeholder ? `placeholder="${sec.placeholder}"` : ''} ${sec.display === false ? 'style="display: none;"' : ''}> ${sec.buttons ? sec.buttons.map(btn => `<button id="${btn.id}" style="margin-top: 8px; padding: 8px 16px; background-color: ${btn.bgColor}; color: white; border-radius: 6px; font-size: 12px;">${btn.text}</button>`).join('') : ''} </div> `; }).join(''); } function bindEvents() { $('#crawler-minimize').on('click', toggleMinimize); $('.crawler-tab').on('click', switchTab); $('#save-settings').on('click', saveSettingsHandler); $('#reset-settings').on('click', resetSettingsHandler); $('#add-user-agent').on('click', addUserAgent); $('#remove-user-agent').on('click', removeUserAgent); $('#download-csv').on('click', Download.downloadCSV); $('#download-json').on('click', Download.downloadJSON); $('#download-xml').on('click', Download.downloadXML); $('#download-xls').on('click', Download.downloadXLS); $('#search-filter').on('input', filterData); $('#schedule-enabled').on('change', toggleScheduleOptions); $('#data-visualization-checkbox').on('change', toggleDataVisualization); $('#enable-captchas-checkbox').on('change', toggleCaptchaHandling); $('#save-template').on('click', saveTemplateHandler); $('#load-template').on('click', loadTemplateHandler); $('#delete-template').on('click', deleteTemplateHandler); $('#template-select').on('change', selectTemplateHandler); $('#run-tests').on('click', Testing.runTests); $('#clear-previous-crawls').on('click', clearPreviousCrawls); $('#reset-data').on('click', resetData); $('#dark-mode-toggle').on('click', toggleDarkMode); $('.close').on('click', () => $('.modal').hide()); $(window).on('click', e => { if ($(e.target).hasClass('modal')) $('.modal').hide(); }); $('#start-crawling').on('click', Crawling.startCrawling); $('#stop-crawling').on('click', Crawling.stopCrawling); $(document).on('keydown', handleShortcuts); $('#proxy-input').on('input', validateProxyInput); $('input[type="checkbox"], input[type="radio"]').on('change', toggleDataField); } function toggleMinimize() { $('#crawler-content').toggle(); $('#crawler-minimize').text($('#crawler-content').is(':hidden') ? '☐' : '─'); Logger.log(`크롤러 창이 ${$('#crawler-content').is(':hidden') ? '최소화' : '확대'}되었습니다.`, 'info'); } function switchTab() { const tab = $(this).data('tab'); $('.crawler-tab').removeClass('active'); $(this).addClass('active'); $('#crawler-content > div').hide(); $(`#tab-${tab}`).show(); Config.saveSettings({ currentTab: tab }); } function updateUI() { const currentTab = Config.settings.currentTab || 'logs'; $(`.crawler-tab[data-tab="${currentTab}"]`).addClass('active'); $('#crawler-content > div').hide(); $(`#tab-${currentTab}`).show(); updateLogs(); updateDataTable(); updatePreviousCrawlsUI(); updateTemplatesUI(); if (Config.settings.dataVisualization) DataVisualization.renderChart(); applyDarkMode(); } function updateLogs() { const logs = Logger.getLogs(); $('#log-messages').html(logs.map(l => `<div>[${l.timestamp}] [${l.type.toUpperCase()}] ${Utils.escape(l.message)}</div>`).join('')); const progress = GM_getValue('progress', 0); const currentPage = GM_getValue('currentPage', 0); const totalPages = Config.settings.maxPages; $('.progress-bar-fill').css('width', `${progress.toFixed(2)}%`); $('.status-text').text(`진행률: ${progress.toFixed(2)}% (${currentPage}/${totalPages} 페이지)`); const latestLog = logs[logs.length -1]; if (latestLog) { if (['error', 'warning'].includes(latestLog.type)) { $('#error-message').text(latestLog.message).show().delay(5000).fadeOut(); } if (['success', 'info'].includes(latestLog.type)) { $('#success-message').text(latestLog.message).show().delay(5000).fadeOut(); } } } function updateDataTable() { const data = JSON.parse(GM_getValue('data', '[]')); const filter = $('#search-filter').val().toLowerCase(); const filtered = filter ? data.filter(item => Object.keys(Config.settings.dataFields).some(f => Config.settings.dataFields[f] && String(item[f]).toLowerCase().includes(filter))) : data; $('#data-table-body').html(filtered.map(item => ` <tr> ${Config.settings.dataFields.name ? `<td>${Utils.escape(item.name)}</td>` : ''} ${Config.settings.dataFields.url ? `<td><a href="${Utils.escape(item.url)}" target="_blank" class="full-url">${Utils.escape(item.url)}</a></td>` : ''} ${Config.settings.dataFields.description ? `<td>${Utils.escape(item.description)}</td>` : ''} ${Config.settings.dataFields.html ? `<td><button class="view-html" data-html="${encodeURIComponent(item.html)}">📄 보기</button></td>` : ''} ${Config.settings.dataFields.category ? `<td>${Utils.escape(item.category)}</td>` : ''} ${Config.settings.dataFields.quality ? `<td>${Utils.escape(item.quality.toString())}</td>` : ''} ${Config.settings.dataFields.language ? `<td>${Utils.escape(item.language)}</td>` : ''} ${Config.settings.dataFields.wordCount ? `<td>${Utils.escape(item.wordCount.toString())}</td>` : ''} ${Config.settings.dataFields.metaTags ? `<td>${Utils.escape(JSON.stringify(item.metaTags))}</td>` : ''} ${Config.settings.dataFields.headings ? `<td>${Utils.escape(item.headings.join(', '))}</td>` : ''} </tr> `).join('')); $('.view-html').off('click').on('click', function() { const htmlContent = decodeURIComponent($(this).data('html')); $('#modal-html-content').text(htmlContent); const codeBlock = document.getElementById('modal-html-content'); hljs.highlightElement(codeBlock); $('#html-modal').show(); }); if (Config.settings.dataVisualization) DataVisualization.renderChart(); } function updatePreviousCrawlsUI() { const previousCrawls = JSON.parse(GM_getValue('previousCrawls', '[]')); $('#previous-crawls-body').html(previousCrawls.map(crawl => ` <tr> <td>${Utils.escape(crawl.date)}</td> <td>${Utils.escape(crawl.count.toString())}</td> <td>${Utils.escape(crawl.status)}</td> </tr> `).join('')); } function updateTemplatesUI() { $('#template-select').empty().append(`<option value="" disabled selected>템플릿 선택</option>`); Object.keys(Config.settings.templates).forEach(t => $('#template-select').append(`<option value="${t}">${t}</option>`)); } function addUserAgent() { const ua = prompt("새 사용자 에이전트를 입력하세요:"); if (ua) { Config.settings.userAgents.push(ua); Config.saveSettings({ userAgents: Config.settings.userAgents }); updateUserAgentSelect(); Logger.log('새 사용자 에이전트가 추가되었습니다.', 'success'); } } function removeUserAgent() { const selected = $('#user-agent-select').val(); if (selected && selected.length) { selected.sort((a, b) => b - a).forEach(idx => Config.settings.userAgents.splice(idx, 1)); Config.settings.userAgentIndex = 0; Config.saveSettings({ userAgents: Config.settings.userAgents, userAgentIndex: Config.settings.userAgentIndex }); updateUserAgentSelect(); Logger.log('선택한 사용자 에이전트가 제거되었습니다.', 'success'); } else { alert('제거할 사용자 에이전트를 선택하세요.'); } } function updateUserAgentSelect() { $('#user-agent-select').empty(); Config.settings.userAgents.forEach((ua, idx) => { $('#user-agent-select').append(`<option value="${idx}" ${idx === Config.settings.userAgentIndex ? 'selected' : ''}>${ua}</option>`); }); } function saveSettingsHandler() { const newSettings = { delay: parseInt($('#delay-input').val(), 10) || Config.DEFAULT_SETTINGS.delay, delayRandom: parseInt($('#delay-random-input').val(), 10) || Config.DEFAULT_SETTINGS.delayRandom, maxPages: parseInt($('#max-pages-input').val(), 10) || Config.DEFAULT_SETTINGS.maxPages, userAgentIndex: parseInt($('#user-agent-select').val(), 10) || 0, contentTypes: { include: $('#content-include-input').val().split(',').map(ct => ct.trim()).filter(Boolean), exclude: $('#content-exclude-input').val().split(',').map(ct => ct.trim()).filter(Boolean) }, timeout: parseInt($('#timeout-input').val(), 10) || Config.DEFAULT_SETTINGS.timeout, includeKeywords: $('#include-keywords-input').val().split('\n').map(k => k.trim()).filter(Boolean), excludeKeywords: $('#exclude-keywords-input').val().split('\n').map(k => k.trim()).filter(Boolean), regexInclude: $('#regex-include-input').val().split('\n').map(r => r.trim()).filter(Boolean), regexExclude: $('#regex-exclude-input').val().split('\n').map(r => r.trim()).filter(Boolean), language: $('#language-select').val() || [], minWordCount: parseInt($('#min-word-count-input').val(), 10) || Config.DEFAULT_SETTINGS.minWordCount, maxWordCount: parseInt($('#max-word-count-input').val(), 10) || Config.DEFAULT_SETTINGS.maxWordCount, maxDepth: parseInt($('#max-depth-input').val(), 10) || Config.DEFAULT_SETTINGS.maxDepth, proxy: $('#proxy-input').val().trim(), dataFields: Object.fromEntries(['name','url','description','html','category','quality','language','wordCount','metaTags','headings'].map(f => [`${f}`, $(`#field-${f}`).is(':checked') ])), retryAttempts: parseInt($('#retry-attempts-input').val(), 10) || Config.DEFAULT_SETTINGS.retryAttempts, retryDelay: parseInt($('#retry-delay-input').val(), 10) || Config.DEFAULT_SETTINGS.retryDelay, schedule: { enabled: $('#schedule-enabled').is(':checked'), startTime: $('#start-time-input').val() || Config.DEFAULT_SETTINGS.schedule.startTime, interval: (parseInt($('#interval-input').val(), 10) || (Config.DEFAULT_SETTINGS.schedule.interval / 60000)) * 60000, specificDays: $('#specific-days-input').val() ? $('#specific-days-input').val().map(d => parseInt(d, 10)) : [] }, searchEngines: Object.fromEntries(Object.keys(Config.settings.searchEngines).map(engine => [`${engine}`, $(`#engine-${engine}`).is(':checked') ])), blacklist: $('#blacklist-input').val().split('\n').map(d => d.trim()).filter(Boolean), whitelist: $('#whitelist-input').val().split('\n').map(d => d.trim()).filter(Boolean), respectRobots: $('#respect-robots-checkbox').is(':checked'), maxConcurrentRequests: parseInt($('#max-concurrent-requests-input').val(), 10) || Config.DEFAULT_SETTINGS.maxConcurrentRequests, dataVisualization: $('#data-visualization-checkbox').is(':checked'), exportFormats: Object.fromEntries(['csv','json','xml','xls'].map(fmt => [`${fmt}`, $(`#export-${fmt}`).is(':checked') ])), advancedLogging: $('#advanced-logging-checkbox').is(':checked'), enableCaptchas: $('#enable-captchas-checkbox').is(':checked'), customKeywords: Utils.parseCustomKeywords($('#custom-keywords-input').val()) }; if (newSettings.proxy && !Config.validateProxy(newSettings.proxy)) { $('#error-message').text('유효하지 않은 프록시 URL입니다.').show(); setTimeout(() => $('#error-message').fadeOut(), 5000); return; } Config.saveSettings(newSettings); Crawling.updateCrawlSettings(); Logger.log('설정이 저장되었습니다.', 'success'); if (newSettings.schedule.enabled) Scheduler.scheduleCrawling(); updateDataTable(); if (newSettings.dataVisualization) DataVisualization.renderChart(); } function resetSettingsHandler() { if (confirm("기본 설정으로 초기화하시겠습니까?")) { Config.resetSettings(); location.reload(); } } function toggleScheduleOptions() { $('#start-time-input, #interval-input, #specific-days-input').toggle($('#schedule-enabled').is(':checked')); } function toggleDataVisualization() { const enabled = $('#data-visualization-checkbox').is(':checked'); Config.saveSettings({ dataVisualization: enabled }); $('#data-visualization').toggle(enabled); Logger.log(`데이터 시각화가 ${enabled ? '활성화' : '비활성화'}되었습니다.`, 'success'); if (enabled) DataVisualization.renderChart(); } function toggleCaptchaHandling() { const enabled = $('#enable-captchas-checkbox').is(':checked'); Config.saveSettings({ enableCaptchas: enabled }); Logger.log(`CAPTCHA 처리 ${enabled ? '활성화' : '비활성화'}되었습니다.`, 'success'); } function filterData() { updateDataTable(); } function clearPreviousCrawls() { if (confirm("이전 크롤링 기록을 모두 초기화하시겠습니까?")) { GM_setValue('previousCrawls', JSON.stringify([])); updatePreviousCrawlsUI(); Logger.log('이전 크롤링 기록이 초기화되었습니다.', 'success'); } } function resetData() { if (confirm("모든 크롤링 데이터를 초기화하시겠습니까?")) { GM_setValue('data', JSON.stringify([])); GM_setValue('visitedUrls', JSON.stringify([])); updateDataTable(); Logger.log('크롤링 데이터가 초기화되었습니다.', 'success'); } } function handleShortcuts(e) { if (e.ctrlKey) { const actions = { '1': 'logs', '2': 'data', '3': 'settings', '4': 'previous-crawls', '5': 'templates', '6': 'tests', 's': 'save', 'r': 'reset', 'p': 'start', 'q': 'stop', 'c': 'csv', 'j': 'json', 'x': 'xml', 'l': 'xls', 'd': 'dark-mode' }; const action = actions[e.key]; if (action) { e.preventDefault(); switch(action) { case 'logs': $('.crawler-tab[data-tab="logs"]').click(); break; case 'data': $('.crawler-tab[data-tab="data"]').click(); break; case 'settings': $('.crawler-tab[data-tab="settings"]').click(); break; case 'previous-crawls': $('.crawler-tab[data-tab="previous-crawls"]').click(); break; case 'templates': $('.crawler-tab[data-tab="templates"]').click(); break; case 'tests': $('.crawler-tab[data-tab="tests"]').click(); break; case 'save': saveSettingsHandler(); break; case 'reset': resetSettingsHandler(); break; case 'start': if (!Crawling.isCrawling()) Crawling.startCrawling(); break; case 'stop': if (Crawling.isCrawling()) Crawling.stopCrawling(); break; case 'csv': Config.settings.exportFormats.csv && Download.downloadCSV(); break; case 'json': Config.settings.exportFormats.json && Download.downloadJSON(); break; case 'xml': Config.settings.exportFormats.xml && Download.downloadXML(); break; case 'xls': Config.settings.exportFormats.xls && Download.downloadXLS(); break; case 'dark-mode': toggleDarkMode(); break; } } } } function makeDraggable() { const container = document.getElementById('crawler-container'); const header = document.getElementById('crawler-header'); let isDragging = false, startX, startY, initialX, initialY; header.onmousedown = function(e) { isDragging = true; startX = e.clientX; startY = e.clientY; initialX = container.offsetLeft; initialY = container.offsetTop; document.onmousemove = drag; document.onmouseup = stopDrag; return false; }; function drag(e) { if (isDragging) { container.style.left = (initialX + e.clientX - startX) + 'px'; container.style.top = (initialY + e.clientY - startY) + 'px'; } } function stopDrag() { isDragging = false; document.onmousemove = null; document.onmouseup = null; } } function getFieldName(field) { const names = { name: '이름', url: '주소', description: '설명', html: 'HTML 내용', category: '분류', quality: '품질점수', language: '언어', wordCount: '단어 수', metaTags: '메타 태그', headings: '헤딩' }; return names[field] || field; } function saveTemplateHandler() { const templateName = $('#new-template-name').val().trim(); if (!templateName) { alert('템플릿 이름을 입력하세요.'); return; } Config.saveTemplate(templateName); updateTemplatesUI(); $('#new-template-name').val(''); } function loadTemplateHandler() { const selected = $('#template-select').val(); if (!selected) { alert('불러올 템플릿을 선택하세요.'); return; } Config.loadTemplate(selected); updateUI(); $('.crawler-tab[data-tab="settings"]').click(); } function deleteTemplateHandler() { const selected = $('#template-select').val(); if (!selected) { alert('삭제할 템플릿을 선택하세요.'); return; } delete Config.settings.templates[selected]; Config.saveSettings({ templates: Config.settings.templates }); Logger.log(`템플릿 ${selected}이(가) 삭제되었습니다.`, 'success'); updateTemplatesUI(); } function selectTemplateHandler() { const selected = $(this).val(); if (selected) { Config.loadTemplate(selected); updateUI(); $('.crawler-tab[data-tab="settings"]').click(); } } function toggleDarkMode() { $('#crawler-container').toggleClass('dark'); const isDark = $('#crawler-container').hasClass('dark'); $('#dark-mode-toggle').text(isDark ? '☀️' : '🌙'); Config.saveSettings({ darkMode: isDark }); Logger.log(`다크 모드가 ${isDark ? '활성화' : '비활성화'}되었습니다.`, 'success'); } function applyDarkMode() { if (Config.settings.darkMode) { $('#crawler-container').addClass('dark'); $('#dark-mode-toggle').text('☀️'); } else { $('#crawler-container').removeClass('dark'); $('#dark-mode-toggle').text('🌙'); } } function toggleDataField() { const field = $(this).attr('id').replace('field-', ''); Config.settings.dataFields[field] = $(this).is(':checked'); Config.saveSettings({ dataFields: Config.settings.dataFields }); Download.updateExportButtons(); } function validateProxyInput() { const proxy = $('#proxy-input').val().trim(); if (proxy && !Config.validateProxy(proxy)) { $('#error-message').text('유효하지 않은 프록시 URL입니다.').show(); } else { $('#error-message').hide(); } } return { createUI, updateUI, updateLogs, updateDataTable, updatePreviousCrawlsUI, updateTemplatesUI, applyDarkMode }; })();