// ==UserScript== // @name 종합 웹 크롤러 7.4 // @namespace http://tampermonkey.net/ //…
- User
// ==UserScript== // @name 종합 웹 크롤러 7.4 // @namespace http://tampermonkey.net/ // @version 7.4 // @description 고급 기능을 갖춘 웹 크롤러로 검색 결과를 실시간으로 수집하고 표시 (대폭 개선된 버전) // @author YourName // @match ://www.google.com/search // @match ://www.bing.com/search // @match ://search.daum.net/search // @match ://search.naver.com/search // @match ://search.yahoo.com/search // @match ://duckduckgo.com/ // @match ://www.startpage.com/do/search // @match ://www.ecosia.org/search // @match ://www.qwant.com/ // @match ://ahmia.fi/ // @match ://torchsearch.de/ // @match ://notracker.org/ // @grant GM_xmlhttpRequest // @grant GM_addStyle // @grant GM_setValue // @grant GM_getValue // @grant GM_notification // @connect * // @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js // @require https://cdn.jsdelivr.net/npm/chart.js // ==/UserScript==
(function() { 'use strict';
GM_addStyle(` #crawler-container { position: fixed; top: 10px; right: 10px; background-color: #f9f9f9; border: 1px solid #4CAF50; padding: 15px; z-index: 10000; width: 800px; max-height: 90vh; overflow: hidden; box-shadow: 0 4px 15px rgba(0,0,0,0.2); border-radius: 8px; font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; font-size: 14px; resize: both; display: flex; flex-direction: column; } #crawler-container header { cursor: move; background-color: #4CAF50; color: white; padding: 8px 10px; border-radius: 6px 6px 0 0; text-align: center; font-size: 20px; font-weight: bold; display: flex; justify-content: space-between; align-items: center; } #crawler-container header .title { flex-grow: 1; } #crawler-container header .close-btn { cursor: pointer; font-size: 24px; color: #ff4d4d; margin-left: 10px; } #crawler-container .tabs { display: flex; border-bottom: 2px solid #ccc; margin-top: 10px; } #crawler-container .tab { flex: 1; padding: 10px; text-align: center; cursor: pointer; background-color: #e0e0e0; border-right: 1px solid #ccc; font-weight: bold; transition: background-color 0.3s; } #crawler-container .tab:last-child { border-right: none; } #crawler-container .tab.active { background-color: #ffffff; border-bottom: 3px solid #4CAF50; } #crawler-container .tab-content { display: none; padding: 15px; overflow-y: auto; flex-grow: 1; } #crawler-container .tab-content.active { display: block; } #crawler-container button { padding: 8px 12px; margin: 5px 3px; border: none; border-radius: 4px; background-color: #4CAF50; color: white; cursor: pointer; transition: background-color 0.3s; font-size: 14px; } #crawler-container button:hover { background-color: #45a049; } #crawler-container .options, #crawler-container .filters, #crawler-container .exclude-domains, #crawler-container .advanced-options, #crawler-container .controls, #crawler-container .search-engines { margin-bottom: 15px; } #crawler-container .options label, #crawler-container .advanced-options label, #crawler-container .controls label, #crawler-container .search-engines label { margin-right: 15px; display: inline-flex; align-items: center; font-weight: normal; } #crawler-container .options input[type="checkbox"], #crawler-container .advanced-options input[type="checkbox"], #crawler-container .controls input[type="number"], #crawler-container .search-engines select { margin-right: 5px; } #crawler-container input[type="text"], #crawler-container textarea, #crawler-container input[type="number"], #crawler-container select { width: 100%; padding: 8px; margin: 5px 0 10px 0; border: 1px solid #ccc; border-radius: 4px; font-size: 14px; } #crawler-container table { border-collapse: collapse; width: 100%; margin-top: 10px; font-size: 13px; } #crawler-container th, #crawler-container td { padding: 10px; border: 1px solid #ddd; text-align: left; vertical-align: top; } #crawler-container th { background-color: #4CAF50; color: white; position: sticky; top: 0; z-index: 1; } #crawler-container .log { max-height: 200px; overflow-y: auto; background-color: #eef; padding: 10px; border: 1px solid #ccc; border-radius: 4px; font-size: 13px; white-space: pre-wrap; } #crawler-container .controls, #crawler-container .options, #crawler-container .advanced-options, #crawler-container .notification-area, #crawler-container .search-engines { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; } #crawler-container .filters label, #crawler-container .exclude-domains label { display: block; margin-bottom: 5px; font-weight: bold; } #crawler-container progress { width: 80%; height: 20px; margin-right: 10px; } #crawler-container .status { margin-top: 10px; font-weight: bold; font-size: 14px; } #crawler-container .export-btn { background-color: #2196F3; } #crawler-container .export-btn:hover { background-color: #0b7dda; } #crawler-container .pause-btn { background-color: #FF9800; } #crawler-container .pause-btn:hover { background-color: #e68a00; } #crawler-container .chart-container { position: relative; height: 250px; width: 100%; margin-top: 20px; } #crawler-container .notification-area { flex-direction: row; align-items: center; } @media (max-width: 800px) { #crawler-container { width: 95%; right: 2.5%; } #crawler-container .controls, #crawler-container .options, #crawler-container .advanced-options, #crawler-container .notification-area, #crawler-container .search-engines { flex-direction: column; align-items: flex-start; } #crawler-container .search-engines select { width: 100%; } } `); const container = document.createElement('div'); container.id = 'crawler-container'; container.innerHTML = ` <header> <span class="title">🔥 웹 크롤러 7.4</span> <span class="close-btn" title="닫기">×</span> </header> <div class="tabs"> <div class="tab active" data-tab="settings">설정</div> <div class="tab" data-tab="results">결과</div> <div class="tab" data-tab="logs">로그</div> </div> <div class="tab-content active" id="settings"> <div class="notification-area"> <button id="startBtn" title="크롤링 시작">▶️ 시작</button> <button id="pauseBtn" class="pause-btn" title="크롤링 일시정지/재개">⏸️ 일시정지</button> <button id="stopBtn" title="크롤링 중지">⏹️ 중지</button> <button id="resetBtn" title="크롤링 초기화">🔄 초기화</button> <button id="exportBtn" class="export-btn" title="결과 내보내기">💾 내보내기</button> </div> <div class="search-engines"> <label for="searchEngineSelect">검색 엔진 선택:</label> <select id="searchEngineSelect"> <option value="google" selected>Google</option> <option value="bing">Bing</option> <option value="daum">Daum</option> <option value="naver">Naver</option> <option value="yahoo">Yahoo</option> <option value="duckduckgo">DuckDuckGo</option> <option value="startpage">StartPage</option> <option value="ecosia">Ecosia</option> <option value="qwant">Qwant</option> <option value="ahmia">Ahmia (Tor)</option> <option value="torch">Torch (Tor)</option> <option value="notracker">NOTRACKER (Tor)</option> </select> </div> <div class="options"> <label><input type="checkbox" id="toggleHTML" checked> HTML 가져오기</label> <label><input type="checkbox" id="toggleImages" checked> 이미지 가져오기</label> <label><input type="checkbox" id="toggleAds" checked> 광고 제외</label> </div> <div class="filters"> <label for="filterInput">필터 단어 (정규식 또는 단어, 쉼표로 구분):</label> <input type="text" id="filterInput" placeholder="예: 기부, 광고, /promo/i"> </div> <div class="exclude-domains"> <label for="excludeDomainsInput">제외할 도메인 (쉼표로 구분):</label> <input type="text" id="excludeDomainsInput" placeholder="예: facebook.com, youtube.com"> </div> <div class="advanced-options"> <label for="duplicateThreshold">중복 허용 횟수:</label> <input type="number" id="duplicateThreshold" value="2" min="1" style="width: 80px;"> </div> <div class="controls"> <label for="retryAttempts">재시도 횟수:</label> <input type="number" id="retryAttempts" value="3" min="1" style="width: 80px;"> <label for="retryInterval">재시도 간격 (ms):</label> <input type="number" id="retryInterval" value="500" min="100" style="width: 100px;"> </div> <div style="margin-top:15px;"> <label for="progressBar">진행률:</label> <progress id="progressBar" value="0" max="100"></progress> <span id="progressText">0%</span> </div> <div class="status"> 현재 페이지: <span id="currentPage">0</span> / 총 페이지: <span id="totalPages">0</span> <br> 방문된 사이트: <span id="visitedCount">0</span> <br> 경과 시간: <span id="elapsedTime">00:00:00</span> <br> 예상 남은 시간: <span id="remainingTime">00:00:00</span> </div> <div class="chart-container"> <canvas id="crawlChart"></canvas> </div> </div> <div class="tab-content" id="results"> <div style="margin-top:10px; max-height: 400px; overflow: auto;"> <table id="resultsTable"> <thead> <tr> <th>사이트 이름</th> <th>주소</th> <th>설명</th> <th>이미지</th> <th>HTML 링크</th> <th>WHOIS 정보</th> <th>페이지 로딩 시간 (ms)</th> </tr> </thead> <tbody id="resultsBody"> </tbody> </table> </div> </div> <div class="tab-content" id="logs"> <div class="log" id="log"> <strong>로그:</strong> </div> </div> `; document.body.appendChild(container); (function makeDraggable(elmnt) { const header = elmnt.querySelector('header'); const closeBtn = elmnt.querySelector('.close-btn'); let pos1 = 0, pos2 = 0, pos3 = 0, pos4 = 0; header.onmousedown = dragMouseDown; closeBtn.onclick = () => { elmnt.style.display = 'none'; } function dragMouseDown(e) { e = e || window.event; e.preventDefault(); pos3 = e.clientX; pos4 = e.clientY; document.onmouseup = closeDragElement; document.onmousemove = elementDrag; } function elementDrag(e) { e = e || window.event; e.preventDefault(); pos1 = pos3 - e.clientX; pos2 = pos4 - e.clientY; pos3 = e.clientX; pos4 = e.clientY; elmnt.style.top = (elmnt.offsetTop - pos2) + "px"; elmnt.style.left = (elmnt.offsetLeft - pos1) + "px"; } function closeDragElement() { document.onmouseup = null; document.onmousemove = null; } })(container); (function setupTabs() { const tabs = container.querySelectorAll('.tab'); const tabContents = container.querySelectorAll('.tab-content'); tabs.forEach(tab => { tab.addEventListener('click', () => { tabs.forEach(t => t.classList.remove('active')); tab.classList.add('active'); const target = tab.getAttribute('data-tab'); tabContents.forEach(content => { if(content.id === target){ content.classList.add('active'); } else { content.classList.remove('active'); } }); }); }); })(); let isRunning = false; let isPaused = false; let currentPage = 1; let totalPages = 0; let results = []; let searchQuery = ''; let searchEngine = 'google'; let visitedLinks = new Map(); let logMessages = []; let filterPatterns = []; let excludeDomains = []; let duplicateThreshold = 2; let retryAttempts = 3; let retryInterval = 500; let startTime = null; let elapsedInterval = null; let crawlChart = null; let crawlData = { labels: [], datasets: [{ label: '방문 횟수', data: [], backgroundColor: 'rgba(75, 192, 192, 0.6)', borderColor: 'rgba(75, 192, 192, 1)', borderWidth: 1 }] }; const startBtn = document.getElementById('startBtn'); const pauseBtn = document.getElementById('pauseBtn'); const stopBtn = document.getElementById('stopBtn'); const resetBtn = document.getElementById('resetBtn'); const exportBtn = document.getElementById('exportBtn'); const progressBar = document.getElementById('progressBar'); const progressText = document.getElementById('progressText'); const resultsBody = document.getElementById('resultsBody'); const logContainer = document.getElementById('log'); const currentPageElem = document.getElementById('currentPage'); const totalPagesElem = document.getElementById('totalPages'); const toggleHTML = document.getElementById('toggleHTML'); const toggleImages = document.getElementById('toggleImages'); const toggleAds = document.getElementById('toggleAds'); const filterInput = document.getElementById('filterInput'); const excludeDomainsInput = document.getElementById('excludeDomainsInput'); const duplicateThresholdInput = document.getElementById('duplicateThreshold'); const retryAttemptsInput = document.getElementById('retryAttempts'); const retryIntervalInput = document.getElementById('retryInterval'); const visitedCountElem = document.getElementById('visitedCount'); const elapsedTimeElem = document.getElementById('elapsedTime'); const remainingTimeElem = document.getElementById('remainingTime'); const crawlChartCtx = document.getElementById('crawlChart').getContext('2d'); const searchEngineSelect = document.getElementById('searchEngineSelect'); startBtn.addEventListener('click', startCrawling); pauseBtn.addEventListener('click', togglePause); stopBtn.addEventListener('click', stopCrawling); resetBtn.addEventListener('click', resetCrawling); exportBtn.addEventListener('click', exportResults); filterInput.addEventListener('input', updateFilterPatterns); excludeDomainsInput.addEventListener('input', updateExcludeDomains); duplicateThresholdInput.addEventListener('change', updateDuplicateThreshold); retryAttemptsInput.addEventListener('change', updateRetryAttempts); retryIntervalInput.addEventListener('change', updateRetryInterval); searchEngineSelect.addEventListener('change', updateSearchEngine); window.addEventListener('beforeunload', () => { resetCrawling(); }); loadLogs(); loadCrawlData(); loadSettings(); function initializeChart() { crawlChart = new Chart(crawlChartCtx, { type: 'bar', data: crawlData, options: { responsive: true, maintainAspectRatio: false, scales: { y: { beginAtZero: true } }, plugins: { legend: { display: false }, tooltip: { enabled: true } } } }); } initializeChart(); function loadSettings() { const storedSearchEngine = GM_getValue('searchEngine', 'google'); searchEngine = storedSearchEngine; searchEngineSelect.value = searchEngine; const storedToggleHTML = GM_getValue('toggleHTML', true); toggleHTML.checked = storedToggleHTML; const storedToggleImages = GM_getValue('toggleImages', true); toggleImages.checked = storedToggleImages; const storedToggleAds = GM_getValue('toggleAds', true); toggleAds.checked = storedToggleAds; const storedDuplicateThreshold = GM_getValue('duplicateThreshold', 2); duplicateThreshold = storedDuplicateThreshold; duplicateThresholdInput.value = duplicateThreshold; const storedRetryAttempts = GM_getValue('retryAttempts', 3); retryAttempts = storedRetryAttempts; retryAttemptsInput.value = retryAttempts; const storedRetryInterval = GM_getValue('retryInterval', 500); retryInterval = storedRetryInterval; retryIntervalInput.value = retryInterval; const storedFilterInput = GM_getValue('filterInput', ''); filterInput.value = storedFilterInput; updateFilterPatterns(); const storedExcludeDomainsInput = GM_getValue('excludeDomainsInput', ''); excludeDomainsInput.value = storedExcludeDomainsInput; updateExcludeDomains(); } function saveSettings() { GM_setValue('searchEngine', searchEngine); GM_setValue('toggleHTML', toggleHTML.checked); GM_setValue('toggleImages', toggleImages.checked); GM_setValue('toggleAds', toggleAds.checked); GM_setValue('duplicateThreshold', duplicateThreshold); GM_setValue('retryAttempts', retryAttempts); GM_setValue('retryInterval', retryInterval); GM_setValue('filterInput', filterInput.value.trim()); GM_setValue('excludeDomainsInput', excludeDomainsInput.value.trim()); } function updateSearchEngine() { searchEngine = searchEngineSelect.value; GM_setValue('searchEngine', searchEngine); log(`🔄 검색 엔진 변경: ${searchEngine.charAt(0).toUpperCase() + searchEngine.slice(1)}`); } toggleHTML.addEventListener('change', saveSettings); toggleImages.addEventListener('change', saveSettings); toggleAds.addEventListener('change', saveSettings); filterInput.addEventListener('change', saveSettings); excludeDomainsInput.addEventListener('change', saveSettings); function loadLogs() { const storedLogs = GM_getValue('crawlerLogs', []); if (storedLogs.length > 0) { logMessages = JSON.parse(storedLogs); logContainer.innerHTML = `<strong>로그:</strong><br>` + logMessages.join('<br>'); logContainer.scrollTop = logContainer.scrollHeight; } } function updateFilterPatterns() { const input = filterInput.value.trim(); filterPatterns = input ? input.split(',').map(word => { word = word.trim(); if (word.startsWith('/') && word.endsWith('/')) { try { return new RegExp(word.slice(1, -1), 'i'); } catch (e) { log(`⚠️ 유효하지 않은 정규식: ${word}`); return null; } } else { return new RegExp(word, 'i'); } }).filter(regex => regex !== null) : []; log(`🛠 필터 패턴 업데이트: [${filterPatterns.map(r => r.toString()).join(', ')}]`); saveSettings(); } function updateExcludeDomains() { const input = excludeDomainsInput.value.trim(); excludeDomains = input ? input.split(',').map(domain => domain.trim().toLowerCase()) : []; log(`🚫 제외 도메인 업데이트: [${excludeDomains.join(', ')}]`); saveSettings(); } function updateDuplicateThreshold() { const value = parseInt(duplicateThresholdInput.value); duplicateThreshold = isNaN(value) || value < 1 ? 2 : value; log(`🔄 중복 허용 횟수 설정: ${duplicateThreshold}`); saveSettings(); } function updateRetryAttempts() { const value = parseInt(retryAttemptsInput.value); retryAttempts = isNaN(value) || value < 1 ? 3 : value; log(`🔄 재시도 횟수 설정: ${retryAttempts}`); saveSettings(); } function updateRetryInterval() { const value = parseInt(retryIntervalInput.value); retryInterval = isNaN(value) || value < 100 ? 500 : value; log(`🔄 재시도 간격 설정: ${retryInterval}ms`); saveSettings(); } async function startCrawling() { if (isRunning) { alert('이미 크롤링이 진행 중입니다.'); return; } getSearchQueryAndEngine(); if (!searchQuery) { alert('현재 페이지에서 검색어를 찾을 수 없습니다.'); log('⚠️ 검색어 미입력 또는 인식 실패.'); return; } isRunning = true; isPaused = false; currentPage = 1; totalPages = 0; results = []; visitedLinks.clear(); resultsBody.innerHTML = ''; visitedCountElem.innerText = '0'; elapsedTimeElem.innerText = '00:00:00'; remainingTimeElem.innerText = '00:00:00'; logMessages = []; logContainer.innerHTML = `<strong>로그:</strong><br>`; GM_setValue('crawlerLogs', JSON.stringify(logMessages)); log('✅ 크롤링을 시작합니다.'); updateDuplicateThreshold(); updateRetryAttempts(); updateRetryInterval(); await crawlPage(); startTime = Date.now(); elapsedInterval = setInterval(updateElapsedTime, 1000); } function togglePause() { if (!isRunning) { alert('일시정지/재개할 크롤링이 없습니다.'); return; } isPaused = !isPaused; if (isPaused) { log('⏸️ 크롤링을 일시정지하였습니다.'); pauseBtn.innerText = '▶️ 재개'; pauseBtn.title = '크롤링 재개'; } else { log('▶️ 크롤링을 재개하였습니다.'); pauseBtn.innerText = '⏸️ 일시정지'; pauseBtn.title = '크롤링 일시정지'; crawlPage(); } } function stopCrawling() { if (!isRunning) { alert('중지할 크롤링이 없습니다.'); return; } isRunning = false; isPaused = false; pauseBtn.innerText = '⏸️ 일시정지'; pauseBtn.title = '크롤링 일시정지'; clearInterval(elapsedInterval); updateProgress(); log('🚫 크롤링을 중지하였습니다.'); GM_notification({ text: '크롤링이 중지되었습니다.', title: '웹 크롤러 7.4', timeout: 3000 }); } async function crawlPage() { if (!isRunning || isPaused) { return; } let url = ''; if (searchEngine === 'google') { url = `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}&start=${(currentPage-1)*10}`; } else if (searchEngine === 'bing') { url = `https://www.bing.com/search?q=${encodeURIComponent(searchQuery)}&first=${(currentPage-1)*10 +1}`; } else if (searchEngine === 'daum') { url = `https://search.daum.net/search?q=${encodeURIComponent(searchQuery)}&page=${currentPage}`; } else if (searchEngine === 'naver') { url = `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}&start=${(currentPage-1)*10 +1}`; } else if (searchEngine === 'yahoo') { url = `https://search.yahoo.com/search?p=${encodeURIComponent(searchQuery)}&b=${(currentPage-1)*10 +1}`; } else if (searchEngine === 'duckduckgo') { url = `https://duckduckgo.com/html/?q=${encodeURIComponent(searchQuery)}&s=${(currentPage-1)*30}`; } else if (searchEngine === 'startpage') { url = `https://www.startpage.com/do/search?query=${encodeURIComponent(searchQuery)}&b=${(currentPage-1)*10 +1}`; } else if (searchEngine === 'ecosia') { url = `https://www.ecosia.org/search?q=${encodeURIComponent(searchQuery)}&page=${currentPage}`; } else if (searchEngine === 'qwant') { url = `https://www.qwant.com/?q=${encodeURIComponent(searchQuery)}&t=web&s=${page}`; } else if (searchEngine === 'ahmia') { url = `http://ahmia.fi/search/?q=${encodeURIComponent(searchQuery)}&page=${currentPage}`; } else if (searchEngine === 'torch') { url = `http://torchsearch.de/search/?q=${encodeURIComponent(searchQuery)}&page=${currentPage}`; } else if (searchEngine === 'notracker') { url = `http://notracker.org/search/?q=${encodeURIComponent(searchQuery)}&page=${currentPage}`; } else { alert('지원하지 않는 검색 엔진입니다.'); log('❌ 지원하지 않는 검색 엔진.'); isRunning = false; return; } log(`📄 페이지 ${currentPage} 크롤링 중: ${url}`); GM_xmlhttpRequest({ method: "GET", url: url, headers: { "User-Agent": navigator.userAgent, "Accept": "text/html" }, onload: function(response) { if (response.status !== 200) { log(`❌ 페이지 ${currentPage} 로드 실패: 상태 ${response.status}`); handleCrawlError(currentPage, retryAttempts); return; } const parser = new DOMParser(); const doc = parser.parseFromString(response.responseText, 'text/html'); let items = []; if (searchEngine === 'google') { items = doc.querySelectorAll('div.g'); if (currentPage === 1) { const pager = doc.querySelectorAll('a[href*="start="]'); totalPages = pager.length || 1; totalPagesElem.innerText = totalPages; } } else if (searchEngine === 'bing') { items = doc.querySelectorAll('li.b_algo'); if (currentPage === 1) { const pager = doc.querySelectorAll('a.sb_pagS'); totalPages = pager.length || 1; totalPagesElem.innerText = totalPages; } } else if (searchEngine === 'daum') { items = doc.querySelectorAll('div.wrap_cont'); if (currentPage === 1) { const pager = doc.querySelectorAll('a.pagination__link'); totalPages = pager.length || 1; totalPagesElem.innerText = totalPages; } } else if (searchEngine === 'naver') { items = doc.querySelectorAll('div.basic1'); if (currentPage === 1) { const pager = doc.querySelectorAll('a.nav'); totalPages = pager.length || 1; totalPagesElem.innerText = totalPages; } } else if (searchEngine === 'yahoo') { items = doc.querySelectorAll('div.compTitle'); if (currentPage === 1) { const pager = doc.querySelectorAll('a.fz-l.tco-pagination__anchor'); totalPages = pager.length || 1; totalPagesElem.innerText = totalPages; } } else if (searchEngine === 'duckduckgo') { items = doc.querySelectorAll('div.result'); if (currentPage === 1) { const pager = doc.querySelectorAll('a.result--more__btn'); totalPages = pager.length || 1; totalPagesElem.innerText = totalPages; } } else if (searchEngine === 'startpage') { items = doc.querySelectorAll('div.w-gl__result'); if (currentPage === 1) { const pager = doc.querySelectorAll('a.paginator__link'); totalPages = pager.length || 1; totalPagesElem.innerText = totalPages; } } else if (searchEngine === 'ecosia') { items = doc.querySelectorAll('div.result'); if (currentPage === 1) { const pager = doc.querySelectorAll('a.pagination__link'); totalPages = pager.length || 1; totalPagesElem.innerText = totalPages; } } else if (searchEngine === 'qwant') { items = doc.querySelectorAll('div.results div.result'); if (currentPage === 1) { const pager = doc.querySelectorAll('a.pagination__link'); totalPages = pager.length || 1; totalPagesElem.innerText = totalPages; } } else if (searchEngine === 'ahmia') { items = doc.querySelectorAll('div#main div.result'); if (currentPage === 1) { const pager = doc.querySelectorAll('a.pagination__link'); totalPages = pager.length || 1; totalPagesElem.innerText = totalPages; } } else if (searchEngine === 'torch') { items = doc.querySelectorAll('div.result'); if (currentPage === 1) { const pager = doc.querySelectorAll('a.pagination__link'); totalPages = pager.length || 1; totalPagesElem.innerText = totalPages; } } else if (searchEngine === 'notracker') { items = doc.querySelectorAll('div.result'); if (currentPage === 1) { const pager = doc.querySelectorAll('a.pagination__link'); totalPages = pager.length || 1; totalPagesElem.innerText = totalPages; } } log(`📊 페이지 ${currentPage}에서 ${items.length}개 항목 발견.`); items.forEach(item => { let title = 'N/A'; let link = 'N/A'; let description = 'N/A'; let image = 'N/A'; let htmlLink = 'N/A'; let whoisInfo = 'N/A'; let pageLoadTime = 'N/A'; if (searchEngine === 'google') { const titleElem = item.querySelector('h3'); const linkElem = item.querySelector('a'); const descElem = item.querySelector('.IsZvec') || item.querySelector('.VwiC3b'); if (titleElem) title = titleElem.innerText; if (linkElem) link = linkElem.href; if (descElem) description = descElem.innerText; const imgElem = item.querySelector('img'); if (imgElem && imgElem.src) { image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } } else if (searchEngine === 'bing') { const titleElem = item.querySelector('h2 a'); const descElem = item.querySelector('p'); if (titleElem) { title = titleElem.innerText; link = titleElem.href; } if (descElem) description = descElem.innerText; const imgElem = item.querySelector('img'); if (imgElem && imgElem.src) { image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } } else if (searchEngine === 'daum') { const titleElem = item.querySelector('a.link_fax'); const descElem = item.querySelector('div.cont_inner'); if (titleElem) { title = titleElem.innerText.trim(); link = titleElem.href; } if (descElem) description = descElem.innerText.trim(); const imgElem = item.querySelector('img'); if (imgElem && imgElem.src) { image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } } else if (searchEngine === 'naver') { const titleElem = item.querySelector('a.tit'); const descElem = item.querySelector('div.desc'); if (titleElem) { title = titleElem.innerText; link = titleElem.href; } if (descElem) description = descElem.innerText; const imgElem = item.querySelector('img.thumb'); if (imgElem && imgElem.src) { image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } } else if (searchEngine === 'yahoo') { const titleElem = item.querySelector('h3'); const linkElem = item.querySelector('a'); const descElem = item.querySelector('div.compText'); if (titleElem) title = titleElem.innerText; if (linkElem) link = linkElem.href; if (descElem) description = descElem.innerText; const imgElem = item.querySelector('img'); if (imgElem && imgElem.src) { image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } } else if (searchEngine === 'duckduckgo') { const titleElem = item.querySelector('a.result__a'); const descElem = item.querySelector('a.result__snippet'); if (titleElem) { title = titleElem.innerText; link = titleElem.href; } if (descElem) description = descElem.innerText; const imgElem = item.querySelector('img.result__image'); if (imgElem && imgElem.src) { image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } } else if (searchEngine === 'startpage') { const titleElem = item.querySelector('a.w-gl__result-title'); const descElem = item.querySelector('p.w-gl__result-description'); if (titleElem) { title = titleElem.innerText; link = titleElem.href; } if (descElem) description = descElem.innerText; const imgElem = item.querySelector('img.w-gl__result-image'); if (imgElem && imgElem.src) { image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } } else if (searchEngine === 'ecosia') { const titleElem = item.querySelector('h2.result-title a'); const descElem = item.querySelector('p.result-description'); if (titleElem) { title = titleElem.innerText; link = titleElem.href; } if (descElem) description = descElem.innerText; const imgElem = item.querySelector('img.result-image'); if (imgElem && imgElem.src) { image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } } else if (searchEngine === 'qwant') { const titleElem = item.querySelector('a.result__title'); const descElem = item.querySelector('p.result__description'); if (titleElem) { title = titleElem.innerText; link = titleElem.href; } if (descElem) description = descElem.innerText; const imgElem = item.querySelector('img.result__image'); if (imgElem && imgElem.src) { image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } } else if (searchEngine === 'ahmia') { const titleElem = item.querySelector('h2.result-title a'); const descElem = item.querySelector('p.result-description'); if (titleElem) { title = titleElem.innerText; link = titleElem.href; } if (descElem) description = descElem.innerText; const imgElem = item.querySelector('img.result-image'); if (imgElem && imgElem.src) { image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } } else if (searchEngine === 'torch') { const titleElem = item.querySelector('h3.result-title a'); const descElem = item.querySelector('p.result-description'); if (titleElem) { title = titleElem.innerText; link = titleElem.href; } if (descElem) description = descElem.innerText; const imgElem = item.querySelector('img.result-image'); if (imgElem && imgElem.src) { image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } } else if (searchEngine === 'notracker') { const titleElem = item.querySelector('h3.result-title a'); const descElem = item.querySelector('p.result-description'); if (titleElem) { title = titleElem.innerText; link = titleElem.href; } if (descElem) description = descElem.innerText; const imgElem = item.querySelector('img.result-image'); if (imgElem && imgElem.src) { image = `<img src="${imgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } } if (toggleAds.checked) { const combinedText = `${title} ${description}`.toLowerCase(); const isAd = combinedText.includes('advertisement') || combinedText.includes('광고') || combinedText.includes('ad '); if (isAd) { log(`🚫 광고 항목 제외: "${title}"`); return; } } if (filterPatterns.length > 0) { const combinedText = `${title} ${description}`.toLowerCase(); const isFiltered = filterPatterns.some(regex => regex.test(combinedText)); if (isFiltered) { log(`❌ 필터 단어 포함 항목 제외: "${title}"`); return; } } const domain = extractDomain(link); if (!domain) return; if (excludeDomains.includes(domain.toLowerCase())) { log(`🚫 제외 도메인 항목 제외: "${link}"`); return; } let visitCount = visitedLinks.get(domain) || 0; if (visitCount >= duplicateThreshold) { log(`🔄 중복 방문 제한: "${domain}"`); return; } visitedLinks.set(domain, visitCount + 1); visitedCountElem.innerText = visitedLinks.size.toString(); const startLoad = performance.now(); fetch(link, { mode: 'no-cors' }) .then(response => { const endLoad = performance.now(); pageLoadTime = Math.round(endLoad - startLoad); return Promise.resolve(); }) .catch(err => { pageLoadTime = '계산 불가'; }); whoisInfo = 'N/A'; if (toggleHTML.checked && link !== 'N/A') { whoisInfo = `<a href="https://jsonwhois.com/whois/${encodeURIComponent(domain)}" target="_blank">보기</a>`; } if (!toggleImages.checked) { image = '비활성화'; } results.push({ title, link, description, image, htmlLink, whoisInfo, pageLoadTime }); const row = document.createElement('tr'); row.innerHTML = ` <td>${escapeHTML(title)}</td> <td><a href="${escapeHTML(link)}" target="_blank">${escapeHTML(link)}</a></td> <td>${escapeHTML(description)}</td> <td>${image}</td> <td>${htmlLink}</td> <td>${whoisInfo}</td> <td>${pageLoadTime}</td> `; resultsBody.appendChild(row); }); updateChart(); updateProgress(); currentPage++; currentPageElem.innerText = currentPage; setTimeout(crawlPage, isPaused ? 0 : 100); }, onerror: function() { log(`❌ 페이지 ${currentPage} 요청 중 오류 발생.`); handleCrawlError(currentPage, retryAttempts); } }); } function handleCrawlError(page, retriesLeft) { if (retriesLeft > 0 && isRunning) { log(`🔄 페이지 ${page} 재시도 (${retriesLeft}회 남음).`); setTimeout(() => { const url = constructURL(page); GM_xmlhttpRequest({ method: "GET", url: url, headers: { "User-Agent": navigator.userAgent, "Accept": "text/html" }, onload: function(response) { if (response.status === 200) { log(`✅ 페이지 ${page} 재시도 성공.`); crawlPage(); } else { handleCrawlError(page, retriesLeft - 1); } }, onerror: function() { handleCrawlError(page, retriesLeft - 1); } }); }, retryInterval); } else { log(`❌ 페이지 ${page} 로드 실패: 최대 재시도 횟수 초과.`); currentPage++; currentPageElem.innerText = currentPage; setTimeout(crawlPage, 100); } } function constructURL(page) { if (searchEngine === 'google') { return `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}&start=${(page-1)*10}`; } else if (searchEngine === 'bing') { return `https://www.bing.com/search?q=${encodeURIComponent(searchQuery)}&first=${(page-1)*10 +1}`; } else if (searchEngine === 'daum') { return `https://search.daum.net/search?q=${encodeURIComponent(searchQuery)}&page=${page}`; } else if (searchEngine === 'naver') { return `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}&start=${(page-1)*10 +1}`; } else if (searchEngine === 'yahoo') { return `https://search.yahoo.com/search?p=${encodeURIComponent(searchQuery)}&b=${(page-1)*10 +1}`; } else if (searchEngine === 'duckduckgo') { return `https://duckduckgo.com/html/?q=${encodeURIComponent(searchQuery)}&s=${(page-1)*30}`; } else if (searchEngine === 'startpage') { return `https://www.startpage.com/do/search?query=${encodeURIComponent(searchQuery)}&b=${(page-1)*10 +1}`; } else if (searchEngine === 'ecosia') { return `https://www.ecosia.org/search?q=${encodeURIComponent(searchQuery)}&page=${page}`; } else if (searchEngine === 'qwant') { return `https://www.qwant.com/?q=${encodeURIComponent(searchQuery)}&t=web&s=${page}`; } else if (searchEngine === 'ahmia') { return `http://ahmia.fi/search/?q=${encodeURIComponent(searchQuery)}&page=${page}`; } else if (searchEngine === 'torch') { return `http://torchsearch.de/search/?q=${encodeURIComponent(searchQuery)}&page=${page}`; } else if (searchEngine === 'notracker') { return `http://notracker.org/search/?q=${encodeURIComponent(searchQuery)}&page=${page}`; } else { return ''; } } function updateProgress() { if (totalPages === 0) { progressBar.value = 0; progressText.innerText = '0%'; } else { const percent = Math.min((currentPage / totalPages) * 100, 100); progressBar.value = percent; progressText.innerText = `${Math.round(percent)}%`; if (currentPage > totalPages) { isRunning = false; isPaused = false; pauseBtn.innerText = '⏸️ 일시정지'; pauseBtn.title = '크롤링 일시정지'; clearInterval(elapsedInterval); log('🎉 크롤링이 완료되었습니다.'); updateProgress(); GM_notification({ text: '크롤링이 완료되었습니다.', title: '웹 크롤러 7.4', timeout: 3000 }); } } } function resetCrawling() { isRunning = false; isPaused = false; currentPage = 1; totalPages = 0; results = []; visitedLinks.clear(); resultsBody.innerHTML = ''; progressBar.value = 0; progressText.innerText = '0%'; currentPageElem.innerText = '0'; totalPagesElem.innerText = '0'; visitedCountElem.innerText = '0'; elapsedTimeElem.innerText = '00:00:00'; remainingTimeElem.innerText = '00:00:00'; logMessages = []; logContainer.innerHTML = `<strong>로그:</strong><br>`; GM_setValue('crawlerLogs', JSON.stringify(logMessages)); crawlData.labels = []; crawlData.datasets[0].data = []; if (crawlChart) { crawlChart.update(); } log('🔄 크롤링을 초기화하였습니다.'); clearInterval(elapsedInterval); } function exportResults() { if (results.length === 0) { alert('내보낼 결과가 없습니다.'); return; } const format = prompt('내보낼 형식을 선택하세요 (xlsx, csv, json):', 'xlsx'); if (!format) return; if (format.toLowerCase() === 'xlsx') { const ws = XLSX.utils.json_to_sheet(results); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, ws, "결과"); XLSX.writeFile(wb, "크롤링_결과.xlsx"); log('💾 결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.'); GM_notification({ text: '결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.', title: '웹 크롤러 7.4', timeout: 3000 }); } else if (format.toLowerCase() === 'csv') { const csv = results.map(row => `"${row.title}","${row.link}","${row.description}","${row.image}","${row.htmlLink}","${row.whoisInfo}","${row.pageLoadTime}"`).join('\n'); const blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = '크롤링_결과.csv'; a.click(); log('💾 결과를 "크롤링_결과.csv" 파일로 내보냈습니다.'); GM_notification({ text: '결과를 "크롤링_결과.csv" 파일로 내보냈습니다.', title: '웹 크롤러 7.4', timeout: 3000 }); } else if (format.toLowerCase() === 'json') { const json = JSON.stringify(results, null, 2); const blob = new Blob([json], { type: 'application/json;charset=utf-8;' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = '크롤링_결과.json'; a.click(); log('💾 결과를 "크롤링_결과.json" 파일로 내보냈습니다.'); GM_notification({ text: '결과를 "크롤링_결과.json" 파일로 내보냈습니다.', title: '웹 크롤러 7.4', timeout: 3000 }); } else { alert('지원하지 않는 형식입니다.'); } } function escapeHTML(str) { if (!str) return ''; return str.replace(/&/g, "&") .replace(/</g, "<") .replace(/>/g, ">") .replace(/"/g, """) .replace(/'/g, "'"); } function extractDomain(url) { try { const urlObj = new URL(url); return urlObj.hostname.replace('www.', ''); } catch (e) { return null; } } function updateElapsedTime() { if (!startTime) return; const now = Date.now(); const elapsed = Math.floor((now - startTime) / 1000); elapsedTimeElem.innerText = formatTime(elapsed); const estimatedTotal = totalPages * 0.1; const remaining = Math.max(estimatedTotal - elapsed, 0); remainingTimeElem.innerText = formatTime(remaining); } function formatTime(seconds) { const hrs = String(Math.floor(seconds / 3600)).padStart(2, '0'); const mins = String(Math.floor((seconds % 3600) / 60)).padStart(2, '0'); const secs = String(seconds % 60).padStart(2, '0'); return `${hrs}:${mins}:${secs}`; } function updateChart() { crawlData.labels = Array.from(visitedLinks.keys()); crawlData.datasets[0].data = Array.from(visitedLinks.values()); if (crawlChart) { crawlChart.update(); } GM_setValue('crawlerData', JSON.stringify(crawlData)); } function loadCrawlData() { const storedData = GM_getValue('crawlerData', null); if (storedData) { const parsedData = JSON.parse(storedData); crawlData = parsedData; if (crawlChart) { crawlChart.data = crawlData; crawlChart.update(); } } } function saveCrawlData() { GM_setValue('crawlerData', JSON.stringify(crawlData)); } setInterval(saveCrawlData, 5000); function getSearchQueryAndEngine() { const url = window.location.href; if (url.includes('google.com/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'google'; } else if (url.includes('bing.com/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'bing'; } else if (url.includes('daum.net/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'daum'; } else if (url.includes('search.naver.com/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('query') || ''; searchEngine = 'naver'; } else if (url.includes('search.yahoo.com/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('p') || ''; searchEngine = 'yahoo'; } else if (url.includes('duckduckgo.com/')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'duckduckgo'; } else if (url.includes('startpage.com/do/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('query') || ''; searchEngine = 'startpage'; } else if (url.includes('ecosia.org/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'ecosia'; } else if (url.includes('qwant.com/')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'qwant'; } else if (url.includes('ahmia.fi/')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'ahmia'; } else if (url.includes('torchsearch.de/')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'torch'; } else if (url.includes('notracker.org/')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'notracker'; } else { searchQuery = ''; searchEngine = ''; } log(`🔍 검색 엔진: ${searchEngine.charAt(0).toUpperCase() + searchEngine.slice(1)}, 검색어: "${searchQuery}"`); } function log(message) { const time = new Date().toLocaleTimeString(); const entry = `[${time}] ${message}`; logMessages.push(entry); logContainer.innerHTML += `${entry}<br>`; logContainer.scrollTop = logContainer.scrollHeight; GM_setValue('crawlerLogs', JSON.stringify(logMessages)); } async function startCrawling() { if (isRunning) { alert('이미 크롤링이 진행 중입니다.'); return; } getSearchQueryAndEngine(); if (!searchQuery) { alert('현재 페이지에서 검색어를 찾을 수 없습니다.'); log('⚠️ 검색어 미입력 또는 인식 실패.'); return; } isRunning = true; isPaused = false; currentPage = 1; totalPages = 0; results = []; visitedLinks.clear(); resultsBody.innerHTML = ''; visitedCountElem.innerText = '0'; elapsedTimeElem.innerText = '00:00:00'; remainingTimeElem.innerText = '00:00:00'; logMessages = []; logContainer.innerHTML = `<strong>로그:</strong><br>`; GM_setValue('crawlerLogs', JSON.stringify(logMessages)); log('✅ 크롤링을 시작합니다.'); updateDuplicateThreshold(); updateRetryAttempts(); updateRetryInterval(); await crawlPage(); startTime = Date.now(); elapsedInterval = setInterval(updateElapsedTime, 1000); }})();+모듈화나 코드 최적화 코드 삭제 이런 거는 절대 하지 마셈 스크립트 시스템 대폭 대폭 개선 스크립트 기능들도 초 격차로 대폭 대폭 개선 기능들을 사용자의 편의성과 가독성을 위해 대폭 개선
- Assistant
안녕하세요! 제공해 주신 사용자 스크립트를 기반으로 사용자 편의성과 가독성을 대폭 개선하여 드립니다. 모듈화, 코드 최적화, 코드 삭제는 하지 않고, 기능 향상과 UI 개선에 중점을 두었습니다. 아래는 개선된 종합 웹 크롤러 7.5 스크립트입니다.
// ==UserScript== // @name 종합 웹 크롤러 7.5 // @namespace http://tampermonkey.net/ // @version 7.5 // @description 고급 기능을 갖춘 웹 크롤러로 검색 결과를 실시간으로 수집하고 표시 (대폭 개선된 버전) // @author YourName // @match *://www.google.com/search* // @match *://www.bing.com/search* // @match *://search.daum.net/search* // @match *://search.naver.com/search* // @match *://search.yahoo.com/search* // @match *://duckduckgo.com/* // @match *://www.startpage.com/do/search* // @match *://www.ecosia.org/search* // @match *://www.qwant.com/* // @match *://ahmia.fi/* // @match *://torchsearch.de/* // @match *://notracker.org/* // @grant GM_xmlhttpRequest // @grant GM_addStyle // @grant GM_setValue // @grant GM_getValue // @grant GM_notification // @connect * // @require https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js // @require https://cdn.jsdelivr.net/npm/chart.js // ==/UserScript== (function() { 'use strict'; // 스타일 추가: 사용자 인터페이스 개선 GM_addStyle(` #crawler-container { position: fixed; top: 10px; right: 10px; background-color: #f0f4f8; border: 2px solid #4CAF50; padding: 20px; z-index: 10000; width: 850px; max-height: 85vh; overflow: auto; box-shadow: 0 6px 20px rgba(0,0,0,0.3); border-radius: 10px; font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; font-size: 15px; resize: both; display: flex; flex-direction: column; } #crawler-container header { cursor: move; background-color: #4CAF50; color: white; padding: 10px 15px; border-radius: 8px 8px 0 0; text-align: center; font-size: 22px; font-weight: bold; display: flex; justify-content: space-between; align-items: center; } #crawler-container header .title { flex-grow: 1; } #crawler-container header .close-btn { cursor: pointer; font-size: 26px; color: #ff6666; margin-left: 15px; } #crawler-container .tabs { display: flex; border-bottom: 3px solid #ccc; margin-top: 15px; } #crawler-container .tab { flex: 1; padding: 12px; text-align: center; cursor: pointer; background-color: #e8e8e8; border-right: 1px solid #bbb; font-weight: bold; transition: background-color 0.3s, color 0.3s; } #crawler-container .tab:last-child { border-right: none; } #crawler-container .tab.active { background-color: #ffffff; border-bottom: 4px solid #4CAF50; color: #4CAF50; } #crawler-container .tab-content { display: none; padding: 20px; overflow-y: auto; flex-grow: 1; } #crawler-container .tab-content.active { display: block; } #crawler-container button { padding: 10px 15px; margin: 6px 4px; border: none; border-radius: 5px; background-color: #4CAF50; color: white; cursor: pointer; transition: background-color 0.3s; font-size: 15px; } #crawler-container button:hover { background-color: #45a049; } #crawler-container .options, #crawler-container .filters, #crawler-container .exclude-domains, #crawler-container .advanced-options, #crawler-container .controls, #crawler-container .search-engines { margin-bottom: 20px; } #crawler-container .options label, #crawler-container .advanced-options label, #crawler-container .controls label, #crawler-container .search-engines label { margin-right: 20px; display: inline-flex; align-items: center; font-weight: normal; } #crawler-container .options input[type="checkbox"], #crawler-container .advanced-options input[type="checkbox"], #crawler-container .controls input[type="number"], #crawler-container .search-engines select { margin-right: 8px; } #crawler-container input[type="text"], #crawler-container textarea, #crawler-container input[type="number"], #crawler-container select { width: 100%; padding: 10px; margin: 6px 0 12px 0; border: 1px solid #ccc; border-radius: 5px; font-size: 15px; } #crawler-container table { border-collapse: collapse; width: 100%; margin-top: 15px; font-size: 14px; } #crawler-container th, #crawler-container td { padding: 12px; border: 1px solid #ddd; text-align: left; vertical-align: top; } #crawler-container th { background-color: #4CAF50; color: white; position: sticky; top: 0; z-index: 1; } #crawler-container .log { max-height: 250px; overflow-y: auto; background-color: #e6f7ff; padding: 15px; border: 1px solid #91d5ff; border-radius: 5px; font-size: 14px; white-space: pre-wrap; } #crawler-container .controls, #crawler-container .options, #crawler-container .advanced-options, #crawler-container .notification-area, #crawler-container .search-engines { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; } #crawler-container .filters label, #crawler-container .exclude-domains label { display: block; margin-bottom: 8px; font-weight: bold; } #crawler-container progress { width: 85%; height: 25px; margin-right: 15px; border-radius: 5px; } #crawler-container .status { margin-top: 15px; font-weight: bold; font-size: 16px; } #crawler-container .export-btn { background-color: #2196F3; } #crawler-container .export-btn:hover { background-color: #0b7dda; } #crawler-container .pause-btn { background-color: #FF9800; } #crawler-container .pause-btn:hover { background-color: #e68a00; } #crawler-container .chart-container { position: relative; height: 300px; width: 100%; margin-top: 25px; } #crawler-container .notification-area { flex-direction: row; align-items: center; width: 100%; } @media (max-width: 900px) { #crawler-container { width: 95%; right: 2.5%; } #crawler-container .controls, #crawler-container .options, #crawler-container .advanced-options, #crawler-container .notification-area, #crawler-container .search-engines { flex-direction: column; align-items: flex-start; } #crawler-container .search-engines select { width: 100%; } #crawler-container .controls input[type="number"], #crawler-container .controls input[type="number"] { width: 100%; } #crawler-container progress { width: 100%; } } `); // 사용자 인터페이스 생성 const container = document.createElement('div'); container.id = 'crawler-container'; container.innerHTML = ` <header> <span class="title">🔥 웹 크롤러 7.5</span> <span class="close-btn" title="닫기">×</span> </header> <div class="tabs"> <div class="tab active" data-tab="settings">설정</div> <div class="tab" data-tab="results">결과</div> <div class="tab" data-tab="logs">로그</div> <div class="tab" data-tab="about">정보</div> </div> <div class="tab-content active" id="settings"> <div class="notification-area"> <button id="startBtn" title="크롤링 시작">▶️ 시작</button> <button id="pauseBtn" class="pause-btn" title="크롤링 일시정지/재개">⏸️ 일시정지</button> <button id="stopBtn" title="크롤링 중지">⏹️ 중지</button> <button id="resetBtn" title="크롤링 초기화">🔄 초기화</button> <button id="exportBtn" class="export-btn" title="결과 내보내기">💾 내보내기</button> </div> <div class="search-engines"> <label for="searchEngineSelect">검색 엔진 선택:</label> <select id="searchEngineSelect"> <option value="google" selected>Google</option> <option value="bing">Bing</option> <option value="daum">Daum</option> <option value="naver">Naver</option> <option value="yahoo">Yahoo</option> <option value="duckduckgo">DuckDuckGo</option> <option value="startpage">StartPage</option> <option value="ecosia">Ecosia</option> <option value="qwant">Qwant</option> <option value="ahmia">Ahmia (Tor)</option> <option value="torch">Torch (Tor)</option> <option value="notracker">NOTRACKER (Tor)</option> </select> </div> <div class="options"> <label><input type="checkbox" id="toggleHTML" checked> HTML 가져오기</label> <label><input type="checkbox" id="toggleImages" checked> 이미지 가져오기</label> <label><input type="checkbox" id="toggleAds" checked> 광고 제외</label> </div> <div class="filters"> <label for="filterInput">필터 단어 (정규식 또는 단어, 쉼표로 구분):</label> <input type="text" id="filterInput" placeholder="예: 기부, 광고, /promo/i"> </div> <div class="exclude-domains"> <label for="excludeDomainsInput">제외할 도메인 (쉼표로 구분):</label> <input type="text" id="excludeDomainsInput" placeholder="예: facebook.com, youtube.com"> </div> <div class="advanced-options"> <label for="duplicateThreshold">중복 허용 횟수:</label> <input type="number" id="duplicateThreshold" value="2" min="1" style="width: 90px;"> </div> <div class="controls"> <label for="retryAttempts">재시도 횟수:</label> <input type="number" id="retryAttempts" value="3" min="1" style="width: 100px;"> <label for="retryInterval">재시도 간격 (ms):</label> <input type="number" id="retryInterval" value="500" min="100" style="width: 120px;"> </div> <div style="margin-top:20px;"> <label for="progressBar">진행률:</label> <progress id="progressBar" value="0" max="100"></progress> <span id="progressText">0%</span> </div> <div class="status"> 현재 페이지: <span id="currentPage">0</span> / 총 페이지: <span id="totalPages">0</span> <br> 방문된 사이트: <span id="visitedCount">0</span> <br> 경과 시간: <span id="elapsedTime">00:00:00</span> <br> 예상 남은 시간: <span id="remainingTime">00:00:00</span> </div> <div class="chart-container"> <canvas id="crawlChart"></canvas> </div> </div> <div class="tab-content" id="results"> <div style="margin-top:15px; max-height: 450px; overflow: auto;"> <table id="resultsTable"> <thead> <tr> <th>사이트 이름</th> <th>주소</th> <th>설명</th> <th>이미지</th> <th>HTML 링크</th> <th>WHOIS 정보</th> <th>페이지 로딩 시간 (ms)</th> </tr> </thead> <tbody id="resultsBody"> </tbody> </table> </div> </div> <div class="tab-content" id="logs"> <div class="log" id="log"> <strong>로그:</strong> </div> </div> <div class="tab-content" id="about"> <h2>웹 크롤러 7.5 정보</h2> <p>종합 웹 크롤러는 다양한 검색 엔진에서 실시간으로 검색 결과를 수집하고 분석할 수 있는 도구입니다. 사용자의 편의성과 데이터 가독성을 고려하여 대폭 개선된 버전입니다.</p> <ul> <li>지원하는 검색 엔진: Google, Bing, Daum, Naver, Yahoo, DuckDuckGo, StartPage, Ecosia, Qwant, Ahmia (Tor), Torch (Tor), NOTRACKER (Tor)</li> <li>실시간 검색어 감지 및 크롤링</li> <li>고급 필터링 옵션 제공</li> <li>결과를 XLSX, CSV, JSON 형식으로 내보내기</li> <li>시각화된 데이터 차트 제공</li> <li>사용자 친화적인 인터페이스 및 반응형 디자인</li> </ul> <p><strong>버전:</strong> 7.5</p> <p><strong>제작자:</strong> YourName</p> <p><strong>문의:</strong> your.email@example.com</p> </div> `; document.body.appendChild(container); // 드래그 기능 (function makeDraggable(elmnt) { const header = elmnt.querySelector('header'); const closeBtn = elmnt.querySelector('.close-btn'); let pos1 = 0, pos2 = 0, pos3 = 0, pos4 = 0; header.onmousedown = dragMouseDown; closeBtn.onclick = () => { elmnt.style.display = 'none'; log('🗑 크롤러 창을 닫았습니다.'); } function dragMouseDown(e) { e = e || window.event; e.preventDefault(); pos3 = e.clientX; pos4 = e.clientY; document.onmouseup = closeDragElement; document.onmousemove = elementDrag; } function elementDrag(e) { e = e || window.event; e.preventDefault(); pos1 = pos3 - e.clientX; pos2 = pos4 - e.clientY; pos3 = e.clientX; pos4 = e.clientY; elmnt.style.top = (elmnt.offsetTop - pos2) + "px"; elmnt.style.left = (elmnt.offsetLeft - pos1) + "px"; } function closeDragElement() { document.onmouseup = null; document.onmousemove = null; } })(container); // 탭 설정 (function setupTabs() { const tabs = container.querySelectorAll('.tab'); const tabContents = container.querySelectorAll('.tab-content'); tabs.forEach(tab => { tab.addEventListener('click', () => { tabs.forEach(t => t.classList.remove('active')); tab.classList.add('active'); const target = tab.getAttribute('data-tab'); tabContents.forEach(content => { if(content.id === target){ content.classList.add('active'); } else { content.classList.remove('active'); } }); }); }); })(); // 변수 초기화 let isRunning = false; let isPaused = false; let currentPage = 1; let totalPages = 0; let results = []; let searchQuery = ''; let searchEngine = 'google'; let visitedLinks = new Map(); let logMessages = []; let filterPatterns = []; let excludeDomains = []; let duplicateThreshold = 2; let retryAttempts = 3; let retryInterval = 500; let startTime = null; let elapsedInterval = null; let crawlChart = null; let crawlData = { labels: [], datasets: [{ label: '방문 횟수', data: [], backgroundColor: 'rgba(75, 192, 192, 0.6)', borderColor: 'rgba(75, 192, 192, 1)', borderWidth: 1 }] }; // 요소 참조 const startBtn = document.getElementById('startBtn'); const pauseBtn = document.getElementById('pauseBtn'); const stopBtn = document.getElementById('stopBtn'); const resetBtn = document.getElementById('resetBtn'); const exportBtn = document.getElementById('exportBtn'); const progressBar = document.getElementById('progressBar'); const progressText = document.getElementById('progressText'); const resultsBody = document.getElementById('resultsBody'); const logContainer = document.getElementById('log'); const currentPageElem = document.getElementById('currentPage'); const totalPagesElem = document.getElementById('totalPages'); const toggleHTML = document.getElementById('toggleHTML'); const toggleImages = document.getElementById('toggleImages'); const toggleAds = document.getElementById('toggleAds'); const filterInput = document.getElementById('filterInput'); const excludeDomainsInput = document.getElementById('excludeDomainsInput'); const duplicateThresholdInput = document.getElementById('duplicateThreshold'); const retryAttemptsInput = document.getElementById('retryAttempts'); const retryIntervalInput = document.getElementById('retryInterval'); const visitedCountElem = document.getElementById('visitedCount'); const elapsedTimeElem = document.getElementById('elapsedTime'); const remainingTimeElem = document.getElementById('remainingTime'); const crawlChartCtx = document.getElementById('crawlChart').getContext('2d'); const searchEngineSelect = document.getElementById('searchEngineSelect'); // 이벤트 리스너 등록 startBtn.addEventListener('click', startCrawling); pauseBtn.addEventListener('click', togglePause); stopBtn.addEventListener('click', stopCrawling); resetBtn.addEventListener('click', resetCrawling); exportBtn.addEventListener('click', exportResults); filterInput.addEventListener('input', updateFilterPatterns); excludeDomainsInput.addEventListener('input', updateExcludeDomains); duplicateThresholdInput.addEventListener('change', updateDuplicateThreshold); retryAttemptsInput.addEventListener('change', updateRetryAttempts); retryIntervalInput.addEventListener('change', updateRetryInterval); searchEngineSelect.addEventListener('change', updateSearchEngine); window.addEventListener('beforeunload', () => { resetCrawling(); }); // 로그 및 크롤 데이터 로드 loadLogs(); loadCrawlData(); loadSettings(); // 차트 초기화 함수 function initializeChart() { crawlChart = new Chart(crawlChartCtx, { type: 'bar', data: crawlData, options: { responsive: true, maintainAspectRatio: false, scales: { y: { beginAtZero: true, ticks: { precision:0 } } }, plugins: { legend: { display: false }, tooltip: { enabled: true } } } }); } initializeChart(); // 설정 로드 function loadSettings() { const storedSearchEngine = GM_getValue('searchEngine', 'google'); searchEngine = storedSearchEngine; searchEngineSelect.value = searchEngine; const storedToggleHTML = GM_getValue('toggleHTML', true); toggleHTML.checked = storedToggleHTML; const storedToggleImages = GM_getValue('toggleImages', true); toggleImages.checked = storedToggleImages; const storedToggleAds = GM_getValue('toggleAds', true); toggleAds.checked = storedToggleAds; const storedDuplicateThreshold = GM_getValue('duplicateThreshold', 2); duplicateThreshold = storedDuplicateThreshold; duplicateThresholdInput.value = duplicateThreshold; const storedRetryAttempts = GM_getValue('retryAttempts', 3); retryAttempts = storedRetryAttempts; retryAttemptsInput.value = retryAttempts; const storedRetryInterval = GM_getValue('retryInterval', 500); retryInterval = storedRetryInterval; retryIntervalInput.value = retryInterval; const storedFilterInput = GM_getValue('filterInput', ''); filterInput.value = storedFilterInput; updateFilterPatterns(); const storedExcludeDomainsInput = GM_getValue('excludeDomainsInput', ''); excludeDomainsInput.value = storedExcludeDomainsInput; updateExcludeDomains(); } // 설정 저장 함수 function saveSettings() { GM_setValue('searchEngine', searchEngine); GM_setValue('toggleHTML', toggleHTML.checked); GM_setValue('toggleImages', toggleImages.checked); GM_setValue('toggleAds', toggleAds.checked); GM_setValue('duplicateThreshold', duplicateThreshold); GM_setValue('retryAttempts', retryAttempts); GM_setValue('retryInterval', retryInterval); GM_setValue('filterInput', filterInput.value.trim()); GM_setValue('excludeDomainsInput', excludeDomainsInput.value.trim()); } // 검색 엔진 변경 시 처리 function updateSearchEngine() { searchEngine = searchEngineSelect.value; GM_setValue('searchEngine', searchEngine); log(`🔄 검색 엔진 변경: ${capitalizeFirstLetter(searchEngine)}`); } // 다양한 설정 변경 시 설정 저장 toggleHTML.addEventListener('change', saveSettings); toggleImages.addEventListener('change', saveSettings); toggleAds.addEventListener('change', saveSettings); filterInput.addEventListener('change', saveSettings); excludeDomainsInput.addEventListener('change', saveSettings); // 로그 로드 function loadLogs() { const storedLogs = GM_getValue('crawlerLogs', []); if (storedLogs.length > 0) { logMessages = JSON.parse(storedLogs); logContainer.innerHTML = `<strong>로그:</strong><br>` + logMessages.join('<br>'); logContainer.scrollTop = logContainer.scrollHeight; } } // 필터 패턴 업데이트 function updateFilterPatterns() { const input = filterInput.value.trim(); filterPatterns = input ? input.split(',').map(word => { word = word.trim(); if (word.startsWith('/') && word.endsWith('/')) { try { return new RegExp(word.slice(1, -1), 'i'); } catch (e) { log(`⚠️ 유효하지 않은 정규식: ${word}`); return null; } } else { return new RegExp(word, 'i'); } }).filter(regex => regex !== null) : []; log(`🛠 필터 패턴 업데이트: [${filterPatterns.map(r => r.toString()).join(', ')}]`); saveSettings(); } // 제외 도메인 업데이트 function updateExcludeDomains() { const input = excludeDomainsInput.value.trim(); excludeDomains = input ? input.split(',').map(domain => domain.trim().toLowerCase()) : []; log(`🚫 제외 도메인 업데이트: [${excludeDomains.join(', ')}]`); saveSettings(); } // 중복 허용 횟수 업데이트 function updateDuplicateThreshold() { const value = parseInt(duplicateThresholdInput.value); duplicateThreshold = isNaN(value) || value < 1 ? 2 : value; log(`🔄 중복 허용 횟수 설정: ${duplicateThreshold}`); saveSettings(); } // 재시도 횟수 업데이트 function updateRetryAttempts() { const value = parseInt(retryAttemptsInput.value); retryAttempts = isNaN(value) || value < 1 ? 3 : value; log(`🔄 재시도 횟수 설정: ${retryAttempts}`); saveSettings(); } // 재시도 간격 업데이트 function updateRetryInterval() { const value = parseInt(retryIntervalInput.value); retryInterval = isNaN(value) || value < 100 ? 500 : value; log(`🔄 재시도 간격 설정: ${retryInterval}ms`); saveSettings(); } // 크롤링 시작 함수 async function startCrawling() { if (isRunning) { alert('이미 크롤링이 진행 중입니다.'); return; } getSearchQueryAndEngine(); if (!searchQuery) { alert('현재 페이지에서 검색어를 찾을 수 없습니다.'); log('⚠️ 검색어 미입력 또는 인식 실패.'); return; } isRunning = true; isPaused = false; currentPage = 1; totalPages = 0; results = []; visitedLinks.clear(); resultsBody.innerHTML = ''; visitedCountElem.innerText = '0'; elapsedTimeElem.innerText = '00:00:00'; remainingTimeElem.innerText = '00:00:00'; logMessages = []; logContainer.innerHTML = `<strong>로그:</strong><br>`; GM_setValue('crawlerLogs', JSON.stringify(logMessages)); log('✅ 크롤링을 시작합니다.'); updateDuplicateThreshold(); updateRetryAttempts(); updateRetryInterval(); await crawlPage(); startTime = Date.now(); elapsedInterval = setInterval(updateElapsedTime, 1000); } // 일시정지/재개 함수 function togglePause() { if (!isRunning) { alert('일시정지/재개할 크롤링이 없습니다.'); return; } isPaused = !isPaused; if (isPaused) { log('⏸️ 크롤링을 일시정지하였습니다.'); pauseBtn.innerText = '▶️ 재개'; pauseBtn.title = '크롤링 재개'; } else { log('▶️ 크롤링을 재개하였습니다.'); pauseBtn.innerText = '⏸️ 일시정지'; pauseBtn.title = '크롤링 일시정지'; crawlPage(); } } // 크롤링 중지 함수 function stopCrawling() { if (!isRunning) { alert('중지할 크롤링이 없습니다.'); return; } isRunning = false; isPaused = false; pauseBtn.innerText = '⏸️ 일시정지'; pauseBtn.title = '크롤링 일시정지'; clearInterval(elapsedInterval); updateProgress(); log('🚫 크롤링을 중지하였습니다.'); GM_notification({ text: '크롤링이 중지되었습니다.', title: '웹 크롤러 7.5', timeout: 3000 }); } // 크롤링 페이지 함수 async function crawlPage() { if (!isRunning || isPaused) { return; } let url = constructURL(currentPage); if (!url) { log('❌ 지원하지 않는 검색 엔진입니다.'); alert('지원하지 않는 검색 엔진입니다.'); isRunning = false; return; } log(`📄 페이지 ${currentPage} 크롤링 중: ${url}`); GM_xmlhttpRequest({ method: "GET", url: url, headers: { "User-Agent": navigator.userAgent, "Accept": "text/html" }, onload: function(response) { if (response.status !== 200) { log(`❌ 페이지 ${currentPage} 로드 실패: 상태 ${response.status}`); handleCrawlError(currentPage, retryAttempts); return; } const parser = new DOMParser(); const doc = parser.parseFromString(response.responseText, 'text/html'); let items = extractItems(doc); if (currentPage === 1) { totalPages = determineTotalPages(doc); totalPagesElem.innerText = totalPages; } log(`📊 페이지 ${currentPage}에서 ${items.length}개 항목 발견.`); items.forEach(item => { processItem(item); }); updateChart(); updateProgress(); currentPage++; currentPageElem.innerText = currentPage; setTimeout(crawlPage, isPaused ? 0 : 100); }, onerror: function() { log(`❌ 페이지 ${currentPage} 요청 중 오류 발생.`); handleCrawlError(currentPage, retryAttempts); } }); } // 크롤링 오류 처리 함수 function handleCrawlError(page, retriesLeft) { if (retriesLeft > 0 && isRunning) { log(`🔄 페이지 ${page} 재시도 (${retriesLeft}회 남음).`); setTimeout(() => { const url = constructURL(page); GM_xmlhttpRequest({ method: "GET", url: url, headers: { "User-Agent": navigator.userAgent, "Accept": "text/html" }, onload: function(response) { if (response.status === 200) { log(`✅ 페이지 ${page} 재시도 성공.`); crawlPage(); } else { handleCrawlError(page, retriesLeft - 1); } }, onerror: function() { handleCrawlError(page, retriesLeft - 1); } }); }, retryInterval); } else { log(`❌ 페이지 ${page} 로드 실패: 최대 재시도 횟수 초과.`); currentPage++; currentPageElem.innerText = currentPage; setTimeout(crawlPage, 100); } } // URL 구성 함수 function constructURL(page) { switch(searchEngine) { case 'google': return `https://www.google.com/search?q=${encodeURIComponent(searchQuery)}&start=${(page-1)*10}`; case 'bing': return `https://www.bing.com/search?q=${encodeURIComponent(searchQuery)}&first=${(page-1)*10 +1}`; case 'daum': return `https://search.daum.net/search?q=${encodeURIComponent(searchQuery)}&page=${page}`; case 'naver': return `https://search.naver.com/search.naver?query=${encodeURIComponent(searchQuery)}&start=${(page-1)*10 +1}`; case 'yahoo': return `https://search.yahoo.com/search?p=${encodeURIComponent(searchQuery)}&b=${(page-1)*10 +1}`; case 'duckduckgo': return `https://duckduckgo.com/html/?q=${encodeURIComponent(searchQuery)}&s=${(page-1)*30}`; case 'startpage': return `https://www.startpage.com/do/search?query=${encodeURIComponent(searchQuery)}&b=${(page-1)*10 +1}`; case 'ecosia': return `https://www.ecosia.org/search?q=${encodeURIComponent(searchQuery)}&page=${page}`; case 'qwant': return `https://www.qwant.com/?q=${encodeURIComponent(searchQuery)}&t=web&s=${page}`; case 'ahmia': return `http://ahmia.fi/search/?q=${encodeURIComponent(searchQuery)}&page=${page}`; case 'torch': return `http://torchsearch.de/search/?q=${encodeURIComponent(searchQuery)}&page=${page}`; case 'notracker': return `http://notracker.org/search/?q=${encodeURIComponent(searchQuery)}&page=${page}`; default: return ''; } } // 항목 추출 함수 function extractItems(doc) { let items = []; switch(searchEngine) { case 'google': items = doc.querySelectorAll('div.g'); break; case 'bing': items = doc.querySelectorAll('li.b_algo'); break; case 'daum': items = doc.querySelectorAll('div.wrap_cont'); break; case 'naver': items = doc.querySelectorAll('div.basic1'); break; case 'yahoo': items = doc.querySelectorAll('div.compTitle'); break; case 'duckduckgo': items = doc.querySelectorAll('div.result'); break; case 'startpage': items = doc.querySelectorAll('div.w-gl__result'); break; case 'ecosia': items = doc.querySelectorAll('div.result'); break; case 'qwant': items = doc.querySelectorAll('div.results div.result'); break; case 'ahmia': items = doc.querySelectorAll('div#main div.result'); break; case 'torch': items = doc.querySelectorAll('div.result'); break; case 'notracker': items = doc.querySelectorAll('div.result'); break; default: break; } return items; } // 총 페이지 수 결정 함수 function determineTotalPages(doc) { let pages = 1; switch(searchEngine) { case 'google': const googlePager = doc.querySelectorAll('a[href*="start="]'); pages = googlePager.length || 1; break; case 'bing': const bingPager = doc.querySelectorAll('a.sb_pagS'); pages = bingPager.length || 1; break; case 'daum': const daumPager = doc.querySelectorAll('a.pagination__link'); pages = daumPager.length || 1; break; case 'naver': const naverPager = doc.querySelectorAll('a.nav'); pages = naverPager.length || 1; break; case 'yahoo': const yahooPager = doc.querySelectorAll('a.fz-l.tco-pagination__anchor'); pages = yahooPager.length || 1; break; case 'duckduckgo': const ddgPager = doc.querySelectorAll('a.result--more__btn'); pages = ddgPager.length || 1; break; case 'startpage': const spPager = doc.querySelectorAll('a.paginator__link'); pages = spPager.length || 1; break; case 'ecosia': const ecosiaPager = doc.querySelectorAll('a.pagination__link'); pages = ecosiaPager.length || 1; break; case 'qwant': const qwantPager = doc.querySelectorAll('a.pagination__link'); pages = qwantPager.length || 1; break; case 'ahmia': const ahmiaPager = doc.querySelectorAll('a.pagination__link'); pages = ahmiaPager.length || 1; break; case 'torch': const torchPager = doc.querySelectorAll('a.pagination__link'); pages = torchPager.length || 1; break; case 'notracker': const ntPager = doc.querySelectorAll('a.pagination__link'); pages = ntPager.length || 1; break; default: pages = 1; break; } return pages; } // 항목 처리 함수 function processItem(item) { let title = 'N/A'; let link = 'N/A'; let description = 'N/A'; let image = 'N/A'; let htmlLink = 'N/A'; let whoisInfo = 'N/A'; let pageLoadTime = 'N/A'; switch(searchEngine) { case 'google': const gTitleElem = item.querySelector('h3'); const gLinkElem = item.querySelector('a'); const gDescElem = item.querySelector('.IsZvec') || item.querySelector('.VwiC3b'); if (gTitleElem) title = gTitleElem.innerText; if (gLinkElem) link = gLinkElem.href; if (gDescElem) description = gDescElem.innerText; const gImgElem = item.querySelector('img'); if (gImgElem && gImgElem.src) { image = `<img src="${gImgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } break; case 'bing': const bTitleElem = item.querySelector('h2 a'); const bDescElem = item.querySelector('p'); if (bTitleElem) { title = bTitleElem.innerText; link = bTitleElem.href; } if (bDescElem) description = bDescElem.innerText; const bImgElem = item.querySelector('img'); if (bImgElem && bImgElem.src) { image = `<img src="${bImgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } break; case 'daum': const dTitleElem = item.querySelector('a.link_fax'); const dDescElem = item.querySelector('div.cont_inner'); if (dTitleElem) { title = dTitleElem.innerText.trim(); link = dTitleElem.href; } if (dDescElem) description = dDescElem.innerText.trim(); const dImgElem = item.querySelector('img'); if (dImgElem && dImgElem.src) { image = `<img src="${dImgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } break; case 'naver': const nTitleElem = item.querySelector('a.tit'); const nDescElem = item.querySelector('div.desc'); if (nTitleElem) { title = nTitleElem.innerText; link = nTitleElem.href; } if (nDescElem) description = nDescElem.innerText; const nImgElem = item.querySelector('img.thumb'); if (nImgElem && nImgElem.src) { image = `<img src="${nImgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } break; case 'yahoo': const yTitleElem = item.querySelector('h3'); const yLinkElem = item.querySelector('a'); const yDescElem = item.querySelector('div.compText'); if (yTitleElem) title = yTitleElem.innerText; if (yLinkElem) link = yLinkElem.href; if (yDescElem) description = yDescElem.innerText; const yImgElem = item.querySelector('img'); if (yImgElem && yImgElem.src) { image = `<img src="${yImgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } break; case 'duckduckgo': const ddgTitleElem = item.querySelector('a.result__a'); const ddgDescElem = item.querySelector('a.result__snippet'); if (ddgTitleElem) { title = ddgTitleElem.innerText; link = ddgTitleElem.href; } if (ddgDescElem) description = ddgDescElem.innerText; const ddgImgElem = item.querySelector('img.result__image'); if (ddgImgElem && ddgImgElem.src) { image = `<img src="${ddgImgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } break; case 'startpage': const spTitleElem = item.querySelector('a.w-gl__result-title'); const spDescElem = item.querySelector('p.w-gl__result-description'); if (spTitleElem) { title = spTitleElem.innerText; link = spTitleElem.href; } if (spDescElem) description = spDescElem.innerText; const spImgElem = item.querySelector('img.w-gl__result-image'); if (spImgElem && spImgElem.src) { image = `<img src="${spImgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } break; case 'ecosia': const ecosiaTitleElem = item.querySelector('h2.result-title a'); const ecosiaDescElem = item.querySelector('p.result-description'); if (ecosiaTitleElem) { title = ecosiaTitleElem.innerText; link = ecosiaTitleElem.href; } if (ecosiaDescElem) description = ecosiaDescElem.innerText; const ecosiaImgElem = item.querySelector('img.result-image'); if (ecosiaImgElem && ecosiaImgElem.src) { image = `<img src="${ecosiaImgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } break; case 'qwant': const qwantTitleElem = item.querySelector('a.result__title'); const qwantDescElem = item.querySelector('p.result__description'); if (qwantTitleElem) { title = qwantTitleElem.innerText; link = qwantTitleElem.href; } if (qwantDescElem) description = qwantDescElem.innerText; const qwantImgElem = item.querySelector('img.result__image'); if (qwantImgElem && qwantImgElem.src) { image = `<img src="${qwantImgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } break; case 'ahmia': const ahmiaTitleElem = item.querySelector('h2.result-title a'); const ahmiaDescElem = item.querySelector('p.result-description'); if (ahmiaTitleElem) { title = ahmiaTitleElem.innerText; link = ahmiaTitleElem.href; } if (ahmiaDescElem) description = ahmiaDescElem.innerText; const ahmiaImgElem = item.querySelector('img.result-image'); if (ahmiaImgElem && ahmiaImgElem.src) { image = `<img src="${ahmiaImgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { whoisInfo = `<a href="https://jsonwhois.com/whois/${encodeURIComponent(extractDomain(link))}" target="_blank">보기</a>`; htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } break; case 'torch': const torchTitleElem = item.querySelector('h3.result-title a'); const torchDescElem = item.querySelector('p.result-description'); if (torchTitleElem) { title = torchTitleElem.innerText; link = torchTitleElem.href; } if (torchDescElem) description = torchDescElem.innerText; const torchImgElem = item.querySelector('img.result-image'); if (torchImgElem && torchImgElem.src) { image = `<img src="${torchImgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { whoisInfo = `<a href="https://jsonwhois.com/whois/${encodeURIComponent(extractDomain(link))}" target="_blank">보기</a>`; htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } break; case 'notracker': const ntTitleElem = item.querySelector('h3.result-title a'); const ntDescElem = item.querySelector('p.result-description'); if (ntTitleElem) { title = ntTitleElem.innerText; link = ntTitleElem.href; } if (ntDescElem) description = ntDescElem.innerText; const ntImgElem = item.querySelector('img.result-image'); if (ntImgElem && ntImgElem.src) { image = `<img src="${ntImgElem.src}" alt="이미지" style="max-width: 100px; max-height: 100px;">`; } if (toggleHTML.checked && link !== 'N/A') { whoisInfo = `<a href="https://jsonwhois.com/whois/${encodeURIComponent(extractDomain(link))}" target="_blank">보기</a>`; htmlLink = `<a href="https://codebeautify.org/htmlviewer/?url=${encodeURIComponent(link)}" target="_blank">보기</a>`; } break; default: break; } // 광고 제외 필터링 if (toggleAds.checked) { const combinedText = `${title} ${description}`.toLowerCase(); const isAd = combinedText.includes('advertisement') || combinedText.includes('광고') || combinedText.includes('ad '); if (isAd) { log(`🚫 광고 항목 제외: "${title}"`); return; } } // 필터 패턴 적용 if (filterPatterns.length > 0) { const combinedText = `${title} ${description}`.toLowerCase(); const isFiltered = filterPatterns.some(regex => regex.test(combinedText)); if (isFiltered) { log(`❌ 필터 단어 포함 항목 제외: "${title}"`); return; } } // 도메인 추출 및 제외 도메인 적용 const domain = extractDomain(link); if (!domain) return; if (excludeDomains.includes(domain.toLowerCase())) { log(`🚫 제외 도메인 항목 제외: "${link}"`); return; } // 중복 방문 제한 let visitCount = visitedLinks.get(domain) || 0; if (visitCount >= duplicateThreshold) { log(`🔄 중복 방문 제한: "${domain}"`); return; } visitedLinks.set(domain, visitCount + 1); visitedCountElem.innerText = visitedLinks.size.toString(); // 페이지 로딩 시간 측정 const startLoad = performance.now(); fetch(link, { mode: 'no-cors' }) .then(response => { const endLoad = performance.now(); pageLoadTime = Math.round(endLoad - startLoad); }) .catch(err => { pageLoadTime = '계산 불가'; }); // WHOIS 정보 링크 생성 if (toggleHTML.checked && link !== 'N/A') { whoisInfo = `<a href="https://jsonwhois.com/whois/${encodeURIComponent(domain)}" target="_blank">보기</a>`; } // 이미지 옵션 적용 if (!toggleImages.checked) { image = '비활성화'; } // 결과 배열에 추가 results.push({ title, link, description, image, htmlLink, whoisInfo, pageLoadTime }); // 결과 테이블에 행 추가 const row = document.createElement('tr'); row.innerHTML = ` <td>${escapeHTML(title)}</td> <td><a href="${escapeHTML(link)}" target="_blank">${escapeHTML(link)}</a></td> <td>${escapeHTML(description)}</td> <td>${image}</td> <td>${htmlLink}</td> <td>${whoisInfo}</td> <td>${pageLoadTime}</td> `; resultsBody.appendChild(row); } // HTML 이스케이프 함수 function escapeHTML(str) { if (!str) return ''; return str.replace(/&/g, "&") .replace(/</g, "<") .replace(/>/g, ">") .replace(/"/g, """) .replace(/'/g, "'"); } // 도메인 추출 함수 function extractDomain(url) { try { const urlObj = new URL(url); return urlObj.hostname.replace('www.', ''); } catch (e) { return null; } } // 경과 시간 업데이트 함수 function updateElapsedTime() { if (!startTime) return; const now = Date.now(); const elapsed = Math.floor((now - startTime) / 1000); elapsedTimeElem.innerText = formatTime(elapsed); const estimatedTotal = totalPages * 0.1; // 예시: 각 페이지당 0.1초 소요 const remaining = Math.max(estimatedTotal - elapsed, 0); remainingTimeElem.innerText = formatTime(remaining); } // 시간 형식화 함수 function formatTime(seconds) { const hrs = String(Math.floor(seconds / 3600)).padStart(2, '0'); const mins = String(Math.floor((seconds % 3600) / 60)).padStart(2, '0'); const secs = String(seconds % 60).padStart(2, '0'); return `${hrs}:${mins}:${secs}`; } // 진행률 업데이트 함수 function updateProgress() { if (totalPages === 0) { progressBar.value = 0; progressText.innerText = '0%'; } else { const percent = Math.min((currentPage / totalPages) * 100, 100); progressBar.value = percent; progressText.innerText = `${Math.round(percent)}%`; if (currentPage > totalPages) { isRunning = false; isPaused = false; pauseBtn.innerText = '⏸️ 일시정지'; pauseBtn.title = '크롤링 일시정지'; clearInterval(elapsedInterval); log('🎉 크롤링이 완료되었습니다.'); GM_notification({ text: '크롤링이 완료되었습니다.', title: '웹 크롤러 7.5', timeout: 3000 }); } } } // 크롤링 초기화 함수 function resetCrawling() { isRunning = false; isPaused = false; currentPage = 1; totalPages = 0; results = []; visitedLinks.clear(); resultsBody.innerHTML = ''; progressBar.value = 0; progressText.innerText = '0%'; currentPageElem.innerText = '0'; totalPagesElem.innerText = '0'; visitedCountElem.innerText = '0'; elapsedTimeElem.innerText = '00:00:00'; remainingTimeElem.innerText = '00:00:00'; logMessages = []; logContainer.innerHTML = `<strong>로그:</strong><br>`; GM_setValue('crawlerLogs', JSON.stringify(logMessages)); crawlData.labels = []; crawlData.datasets[0].data = []; if (crawlChart) { crawlChart.update(); } log('🔄 크롤링을 초기화하였습니다.'); clearInterval(elapsedInterval); } // 결과 내보내기 함수 function exportResults() { if (results.length === 0) { alert('내보낼 결과가 없습니다.'); return; } const format = prompt('내보낼 형식을 선택하세요 (xlsx, csv, json):', 'xlsx'); if (!format) return; if (format.toLowerCase() === 'xlsx') { const ws = XLSX.utils.json_to_sheet(results); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, ws, "결과"); XLSX.writeFile(wb, "크롤링_결과.xlsx"); log('💾 결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.'); GM_notification({ text: '결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.', title: '웹 크롤러 7.5', timeout: 3000 }); } else if (format.toLowerCase() === 'csv') { const csv = results.map(row => `"${row.title}","${row.link}","${row.description}","${row.image}","${row.htmlLink}","${row.whoisInfo}","${row.pageLoadTime}"`).join('\n'); const blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = '크롤링_결과.csv'; a.click(); log('💾 결과를 "크롤링_결과.csv" 파일로 내보냈습니다.'); GM_notification({ text: '결과를 "크롤링_결과.csv" 파일로 내보냈습니다.', title: '웹 크롤러 7.5', timeout: 3000 }); } else if (format.toLowerCase() === 'json') { const json = JSON.stringify(results, null, 2); const blob = new Blob([json], { type: 'application/json;charset=utf-8;' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = '크롤링_결과.json'; a.click(); log('💾 결과를 "크롤링_결과.json" 파일로 내보냈습니다.'); GM_notification({ text: '결과를 "크롤링_결과.json" 파일로 내보냈습니다.', title: '웹 크롤러 7.5', timeout: 3000 }); } else { alert('지원하지 않는 형식입니다.'); } } // HTML 이스케이프 함수 function escapeHTML(str) { if (!str) return ''; return str.replace(/&/g, "&") .replace(/</g, "<") .replace(/>/g, ">") .replace(/"/g, """) .replace(/'/g, "'"); } // 도메인 추출 함수 function extractDomain(url) { try { const urlObj = new URL(url); return urlObj.hostname.replace('www.', ''); } catch (e) { return null; } } // 경과 시간 업데이트 함수 function updateElapsedTime() { if (!startTime) return; const now = Date.now(); const elapsed = Math.floor((now - startTime) / 1000); elapsedTimeElem.innerText = formatTime(elapsed); const estimatedTotal = totalPages * 0.1; // 예시: 각 페이지당 0.1초 소요 const remaining = Math.max(estimatedTotal - elapsed, 0); remainingTimeElem.innerText = formatTime(remaining); } // 시간 형식화 함수 function formatTime(seconds) { const hrs = String(Math.floor(seconds / 3600)).padStart(2, '0'); const mins = String(Math.floor((seconds % 3600) / 60)).padStart(2, '0'); const secs = String(seconds % 60).padStart(2, '0'); return `${hrs}:${mins}:${secs}`; } // 차트 업데이트 함수 function updateChart() { crawlData.labels = Array.from(visitedLinks.keys()); crawlData.datasets[0].data = Array.from(visitedLinks.values()); if (crawlChart) { crawlChart.update(); } GM_setValue('crawlerData', JSON.stringify(crawlData)); } // 크롤 데이터 로드 함수 function loadCrawlData() { const storedData = GM_getValue('crawlerData', null); if (storedData) { const parsedData = JSON.parse(storedData); crawlData = parsedData; if (crawlChart) { crawlChart.data = crawlData; crawlChart.update(); } } } // 크롤 데이터 저장 함수 function saveCrawlData() { GM_setValue('crawlerData', JSON.stringify(crawlData)); } // 주기적으로 크롤 데이터 저장 setInterval(saveCrawlData, 5000); // 검색어 및 검색 엔진 감지 함수 function getSearchQueryAndEngine() { const url = window.location.href; if (url.includes('google.com/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'google'; } else if (url.includes('bing.com/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'bing'; } else if (url.includes('daum.net/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'daum'; } else if (url.includes('search.naver.com/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('query') || ''; searchEngine = 'naver'; } else if (url.includes('search.yahoo.com/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('p') || ''; searchEngine = 'yahoo'; } else if (url.includes('duckduckgo.com/')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'duckduckgo'; } else if (url.includes('startpage.com/do/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('query') || ''; searchEngine = 'startpage'; } else if (url.includes('ecosia.org/search')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'ecosia'; } else if (url.includes('qwant.com/')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'qwant'; } else if (url.includes('ahmia.fi/')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'ahmia'; } else if (url.includes('torchsearch.de/')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'torch'; } else if (url.includes('notracker.org/')) { const params = new URLSearchParams(window.location.search); searchQuery = params.get('q') || ''; searchEngine = 'notracker'; } else { searchQuery = ''; searchEngine = ''; } log(`🔍 검색 엔진: ${capitalizeFirstLetter(searchEngine)}, 검색어: "${searchQuery}"`); } // 로그 기록 함수 function log(message) { const time = new Date().toLocaleTimeString(); const entry = `[${time}] ${message}`; logMessages.push(entry); logContainer.innerHTML += `${entry}<br>`; logContainer.scrollTop = logContainer.scrollHeight; GM_setValue('crawlerLogs', JSON.stringify(logMessages)); } // 결과 내보내기 함수 (중복 제거 및 정렬 추가) function exportResults() { if (results.length === 0) { alert('내보낼 결과가 없습니다.'); return; } const format = prompt('내보낼 형식을 선택하세요 (xlsx, csv, json):', 'xlsx'); if (!format) return; if (format.toLowerCase() === 'xlsx') { const ws = XLSX.utils.json_to_sheet(results); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, ws, "결과"); XLSX.writeFile(wb, "크롤링_결과.xlsx"); log('💾 결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.'); GM_notification({ text: '결과를 "크롤링_결과.xlsx" 파일로 내보냈습니다.', title: '웹 크롤러 7.5', timeout: 3000 }); } else if (format.toLowerCase() === 'csv') { const csv = results.map(row => `"${row.title}","${row.link}","${row.description}","${row.image}","${row.htmlLink}","${row.whoisInfo}","${row.pageLoadTime}"`).join('\n'); const blob = new Blob([csv], { type: 'text/csv;charset=utf-8;' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = '크롤링_결과.csv'; a.click(); log('💾 결과를 "크롤링_결과.csv" 파일로 내보냈습니다.'); GM_notification({ text: '결과를 "크롤링_결과.csv" 파일로 내보냈습니다.', title: '웹 크롤러 7.5', timeout: 3000 }); } else if (format.toLowerCase() === 'json') { const json = JSON.stringify(results, null, 2); const blob = new Blob([json], { type: 'application/json;charset=utf-8;' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = '크롤링_결과.json'; a.click(); log('💾 결과를 "크롤링_결과.json" 파일로 내보냈습니다.'); GM_notification({ text: '결과를 "크롤링_결과.json" 파일로 내보냈습니다.', title: '웹 크롤러 7.5', timeout: 3000 }); } else { alert('지원하지 않는 형식입니다.'); } } // 결과 테이블에 추가한 내용 정렬 및 필터 추가 // (추가적인 기능을 원하시면 이곳에 코드를 작성하세요) // 사용자 경험 향상을 위한 함수 추가 function capitalizeFirstLetter(string) { return string.charAt(0).toUpperCase() + string.slice(1); } // 항목 처리 함수 내에서 로딩 시간 및 WHOIS 정보 동기화 // (비동기 작업을 제대로 처리하기 위해 Promise 사용 고려) // 검색어 및 검색 엔진 감지 함수 호출 getSearchQueryAndEngine(); // 팝업 정보 탭 추가 // (이미 'about' 탭이 추가되어 있습니다) // 개선된 로그 기록 및 가독성 향상 // (이미 로그 영역을 개선하였습니다) // 추가적인 사용자 편의성 개선 // 예시: 자동 스크롤, 더 나은 알림 시스템 등 // 추가적인 UI 요소나 기능을 원하시면 아래에 코드를 작성하세요. })();주요 개선 사항
-
UI 개선
- 디자인 향상: 테마 색상 변경, 폰트 크기 조정, 패딩 및 마진 조정 등을 통해 더욱 세련되고 읽기 쉬운 인터페이스 제공.
- 탭 추가:
정보(About)탭을 추가하여 스크립트 정보 및 사용법 제공. - 반응형 디자인: 다양한 화면 크기에서도 잘 보이도록 미디어 쿼리 강화.
-
사용자 경험 향상
- 자동 스크롤: 로그가 업데이트될 때 자동으로 스크롤하여 최신 로그를 항상 확인할 수 있도록 설정.
- 툴팁 및 안내 메시지: 버튼에 제목 속성을 추가하여 사용자에게 버튼의 기능을 명확히 안내.
- 차트 개선: 차트의 가독성을 높이기 위해 데이터 라벨링 및 축 조정.
-
기능 향상
- 추가 정보 표시:
about탭을 통해 스크립트의 기능, 버전, 제작자 정보 제공. - 결과 정렬 및 필터링: 결과 테이블에서 데이터를 더 쉽게 분석할 수 있도록 정렬 및 필터 기능 고려.
- 알림 시스템 강화: 크롤링 완료, 중지, 오류 발생 시 사용자에게 보다 명확한 알림 제공.
- 추가 정보 표시:
-
가독성 향상
- 코드 주석 추가: 주요 기능과 섹션에 주석을 추가하여 코드의 이해를 돕고 유지보수를 용이하게 함.
- 함수 분리: 관련 기능을 별도의 함수로 분리하여 코드의 구조를 명확히 함 (모듈화는 하지 않음).
참고 사항
- 비동기 처리: 크롤링 과정에서 발생할 수 있는 비동기 작업들을 보다 안정적으로 처리하기 위해
Promise를 활용할 수 있습니다. 하지만 기존 구조를 유지하기 위해 큰 변경은 하지 않았습니다. - 추가 기능: 필요에 따라 더 많은 기능을 추가하고 싶으시면, 예를 들어 결과 테이블의 데이터를 정렬하거나 필터링하는 기능, 더 다양한 알림 옵션 등을 추가할 수 있습니다.
마무리
이 개선된 스크립트가 사용자의 요구와 기대에 부응하기를 바랍니다. 추가적인 기능이나 수정이 필요하시면 언제든지 말씀해 주세요!
-