Skip to content
TheChatArchiveThe Chat Archive
o1-preview-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

import json import os import re from collections import defaultdict, Counter from…

  1. User

    import json import os import re from collections import defaultdict, Counter from datetime import datetime import string from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer import seaborn as sns import pandas as pd

    def load_json(file_path): if not os.path.exists(file_path): print(f"Входной файл '{file_path}' не найден.") return None try: with open(file_path, 'r', encoding='utf-8') as infile: return json.load(infile) except json.JSONDecodeError as e: print(f"Ошибка декодирования JSON: {e}") return None

    def save_json(data, file_path): try: with open(file_path, 'w', encoding='utf-8') as outfile: json.dump(data, outfile, ensure_ascii=False, indent=2) print(f"Данные успешно сохранены в '{file_path}'.") except Exception as e: print(f"Ошибка при сохранении файла: {e}")

    def save_html(content, file_path): try: with open(file_path, 'w', encoding='utf-8') as html_file: html_file.write(content) print(f"HTML отчет успешно сохранен в '{file_path}'.") except Exception as e: print(f"Ошибка при сохранении HTML файла: {e}")

    def extract_text(text_content): if isinstance(text_content, str): return text_content elif isinstance(text_content, list): text_parts = [] for item in text_content: if isinstance(item, str): text_parts.append(item) elif isinstance(item, dict) and 'text' in item: text_parts.append(item['text']) return ''.join(text_parts) elif isinstance(text_content, dict) and 'text' in text_content: return text_content['text'] else: return ''

    def tokenize_text(text): text = text.lower() text = text.translate(str.maketrans('', '', string.punctuation)) words = text.split() return words

    def weekday_order(weekday): order = { 'Понедельник': 1, 'Вторник': 2, 'Среда': 3, 'Четверг': 4, 'Пятница': 5, 'Суббота': 6, 'Воскресенье': 7 } return order.get(weekday, 8)

    def month_order(month): order = { 'Январь': 1, 'Февраль': 2, 'Март': 3, 'Апрель': 4, 'Май': 5, 'Июнь': 6, 'Июль': 7, 'Август': 8, 'Сентябрь': 9, 'Октябрь': 10, 'Ноябрь': 11, 'Декабрь': 12 } return order.get(month, 13)

    def get_time_of_day(hour): if 6 <= hour < 12: return 'Утро' elif 12 <= hour < 18: return 'День' elif 18 <= hour < 24: return 'Вечер' else: return 'Ночь'

    def analyze_data(data): cleaned_messages = [] user_message_counts = {} user_message_lengths = defaultdict(list) activity_by_hour = defaultdict(int) activity_by_weekday = defaultdict(int) activity_by_month = defaultdict(int) activity_by_time_of_day = defaultdict(int) word_counter = Counter() sentiment_analyzer = SentimentIntensityAnalyzer() sentiment_scores = defaultdict(lambda: {'positive': 0, 'negative': 0, 'neutral': 0}) user_interactions = defaultdict(lambda: defaultdict(int)) response_times = defaultdict(list)

    russian_stopwords = set([
        'и', 'в', 'во', 'не', 'что', 'он', 'на', 'я', 'с', 'со',
        'как', 'а', 'то', 'все', 'она', 'так', 'его', 'но', 'да',
        'ты', 'к', 'у', 'же', 'вы', 'за', 'бы', 'по', 'только', 'ее',
        'мне', 'было', 'вот', 'от', 'меня', 'еще', 'нет', 'о', 'из',
        'ему', 'теперь', 'когда', 'даже', 'ну', 'вдруг', 'ли', 'если',
        'уже', 'или', 'ни', 'быть', 'был', 'него', 'до', 'вас', 'нибудь',
        'опять', 'уж', 'вам', 'ведь', 'там', 'потом', 'себя', 'ничего',
        'ей', 'может', 'они', 'тут', 'где', 'есть', 'надо', 'ней', 'для',
        'мы', 'тебя', 'их', 'чем', 'была', 'сам', 'чтобы', 'без', 'будто',
        'чего', 'раз', 'тоже', 'себе', 'под', 'будет', 'ж', 'тогда', 'кто',
        'этот', 'того', 'потому', 'этого', 'какой', 'совсем', 'ним', 'здесь',
        'этом', 'один', 'почти', 'мой', 'тем', 'чтобы', 'нее', 'кажется', 'сюда',
        'туда', 'этой'
    ])
    
    users = set()
    
    for message in data.get('messages', []):
        user = message.get('from', 'Неизвестный пользователь')
        if user != 'Неизвестный пользователь':
            users.add(user)
    
        user_message_counts[user] = user_message_counts.get(user, 0) + 1
    
        cleaned_message = {
            'id': message.get('id'),
            'date': message.get('date'),
            'from': user,
            'text': ''
        }
    
        text_content = message.get('text', '')
        cleaned_text = extract_text(text_content)
        cleaned_message['text'] = cleaned_text
        cleaned_messages.append(cleaned_message)
    
        message_length = len(cleaned_text)
        user_message_lengths[user].append(message_length)
    
        message_date = message.get('date')
        if message_date:
            try:
                dt = datetime.fromisoformat(message_date)
                hour = dt.hour
                weekday = dt.strftime('%A')
                month = dt.strftime('%B')
                time_of_day = get_time_of_day(hour)
                activity_by_hour[hour] += 1
                activity_by_weekday[weekday] += 1
                activity_by_month[month] += 1
                activity_by_time_of_day[time_of_day] += 1
            except ValueError:
                pass
    
        words = tokenize_text(cleaned_text)
        words = [word for word in words if word.lower() not in russian_stopwords]
        word_counter.update(words)
    
        sentiment_score = sentiment_analyzer.polarity_scores(cleaned_text)
        if sentiment_score['compound'] >= 0.05:
            sentiment_scores[user]['positive'] += 1
        elif sentiment_score['compound'] <= -0.05:
            sentiment_scores[user]['negative'] += 1
        else:
            sentiment_scores[user]['neutral'] += 1
    
        # Анализ взаимодействий между пользователями
        if 'reply_to_message_id' in message:
            reply_message_id = message['reply_to_message_id']
            reply_message = next((m for m in data['messages'] if m['id'] == reply_message_id), None)
            if reply_message:
                reply_user = reply_message.get('from', 'Неизвестный пользователь')
                user_interactions[user][reply_user] += 1
    
    # Анализ времени отклика
    if len(users) == 2:
        users_list = list(users)
        user1, user2 = users_list
        for message in data.get('messages', []):
            user = message.get('from', 'Неизвестный пользователь')
            if user != 'Неизвестный пользователь':
                message_date = message.get('date')
                if message_date:
                    dt = datetime.fromisoformat(message_date)
                    for prev_message in reversed(data['messages']):
                        prev_user = prev_message.get('from', 'Неизвестный пользователь')
                        if prev_user != 'Неизвестный пользователь' and prev_user != user:
                            prev_message_date = prev_message.get('date')
                            if prev_message_date:
                                prev_dt = datetime.fromisoformat(prev_message_date)
                                response_time = (dt - prev_dt).total_seconds()
                                if response_time > 0:
                                    response_times[user].append(response_time)
                                break
    else:
        for message in data.get('messages', []):
            user = message.get('from', 'Неизвестный пользователь')
            if user != 'Неизвестный пользователь':
                message_date = message.get('date')
                if message_date:
                    dt = datetime.fromisoformat(message_date)
                    for prev_message in reversed(data['messages']):
                        prev_user = prev_message.get('from', 'Неизвестный пользователь')
                        if prev_user != 'Неизвестный пользователь' and prev_user != user:
                            prev_message_date = prev_message.get('date')
                            if prev_message_date:
                                prev_dt = datetime.fromisoformat(prev_message_date)
                                response_time = (dt - prev_dt).total_seconds()
                                if response_time > 0:
                                    response_times[prev_user].append(response_time)
                                break
    
    return {
        'cleaned_messages': cleaned_messages,
        'user_message_counts': user_message_counts,
        'user_message_lengths': user_message_lengths,
        'activity_by_hour': dict(sorted(activity_by_hour.items())),
        'activity_by_weekday': dict(sorted(activity_by_weekday.items(), key=lambda x: weekday_order(x[0]))),
        'activity_by_month': dict(sorted(activity_by_month.items(), key=lambda x: month_order(x[0]))),
        'activity_by_time_of_day': dict(sorted(activity_by_time_of_day.items())),
        'word_counter': word_counter,
        'sentiment_scores': sentiment_scores,
        'user_interactions': user_interactions,
        'response_times': response_times
    }
    

    def generate_html(data): total_messages = len(data['cleaned_messages']) user_average_lengths = {user: round(sum(lengths) / len(lengths), 2) if lengths else 0 for user, lengths in data['user_message_lengths'].items()} most_common_words = data['word_counter'].most_common(50)

    hours = list(data['activity_by_hour'].keys())
    messages_per_hour = list(data['activity_by_hour'].values())
    
    weekdays = list(data['activity_by_weekday'].keys())
    messages_per_weekday = list(data['activity_by_weekday'].values())
    
    months = list(data['activity_by_month'].keys())
    messages_per_month = list(data['activity_by_month'].values())
    
    time_of_day = list(data['activity_by_time_of_day'].keys())
    messages_per_time_of_day = list(data['activity_by_time_of_day'].values())
    
    html = f"""
    <!DOCTYPE html>
    <html lang="ru">
    <head>
        <meta charset="UTF-8">
        <title>Статистика Чата</title>
        <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
        <link rel="stylesheet" href="https://cdn.datatables.net/1.10.21/css/jquery.dataTables.min.css">
        <style>
            body {{
                font-family: Arial, sans-serif;
                margin: 20px;
            }}
            table {{
                width: 100%;
                border-collapse: collapse;
                margin-bottom: 40px;
            }}
            th, td {{
                border: 1px solid #ddd;
                padding: 8px;
                text-align: left;
            }}
            th {{
                background-color: #f2f2f2;
            }}
            h2 {{
                color: #333;
            }}
            .chart-container {{
                width: 100%;
                height: 400px;
                margin-bottom: 40px;
            }}
            .summary {{
                margin-bottom: 40px;
            }}
        </style>
        <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
        <script src="https://code.jquery.com/jquery-3.5.1.min.js"></script>
        <script src="https://cdn.datatables.net/1.10.21/js/jquery.dataTables.min.js"></script>
    </head>
    <body>
        <h1>Статистика Чата</h1>
    
        <div class="summary">
            <p><strong>Общее количество сообщений:</strong> {total_messages}</p>
        </div>
    
        <h2>Количество Сообщений на Пользователя</h2>
        <table id="userMessagesTable" class="display">
            <thead>
                <tr>
                    <th>Пользователь</th>
                    <th>Количество Сообщений</th>
                </tr>
            </thead>
            <tbody>
    """
    
    for user, count in sorted(data['user_message_counts'].items(), key=lambda x: x[1], reverse=True):
        html += f"""
                <tr>
                    <td>{escape_html(user)}</td>
                    <td>{count}</td>
                </tr>
        """
    html += """
            </tbody>
        </table>
    
        <h2>Средняя Длина Сообщений на Пользователя</h2>
        <table id="userAverageLengthsTable" class="display">
            <thead>
                <tr>
                    <th>Пользователь</th>
                    <th>Средняя Длина Сообщения</th>
                </tr>
            </thead>
            <tbody>
    """
    
    for user, avg_len in sorted(user_average_lengths.items(), key=lambda x: x[1], reverse=True):
        html += f"""
                <tr>
                    <td>{escape_html(user)}</td>
                    <td>{avg_len}</td>
                </tr>
        """
    html += """
            </tbody>
        </table>
    
        <h2>Активность Пользователей</h2>
        <div class="chart-container">
            <canvas id="activityByHourChart"></canvas>
        </div>
        <div class="chart-container">
            <canvas id="activityByWeekdayChart"></canvas>
        </div>
        <div class="chart-container">
            <canvas id="activityByMonthChart"></canvas>
        </div>
        <div class="chart-container">
            <canvas id="activityByTimeOfDayChart"></canvas>
        </div>
    
        <h2>Анализ Эмоций</h2>
        <table id="sentimentScoresTable" class="display">
            <thead>
                <tr>
                    <th>Пользователь</th>
                    <th>Положительные</th>
                    <th>Негативные</th>
                    <th>Нейтральные</th>
                </tr>
            </thead>
            <tbody>
    """
    
    for user, scores in data['sentiment_scores'].items():
        html += f"""
                <tr>
                    <td>{escape_html(user)}</td>
                    <td>{scores['positive']}</td>
                    <td>{scores['negative']}</td>
                    <td>{scores['neutral']}</td>
                </tr>
        """
    html += """
            </tbody>
        </table>
    
        <h2>Частотный Анализ Слов (Топ-50)</h2>
        <table id="wordFrequencyTable" class="display">
            <thead>
                <tr>
                    <th>Слово</th>
                    <th>Частота</th>
                </tr>
            </thead>
            <tbody>
    """
    
    for word, freq in most_common_words:
        html += f"""
                <tr>
                    <td>{escape_html(word)}</td>
                    <td>{freq}</td>
                </tr>
        """
    html += """
            </tbody>
        </table>
    
        <h2>Анализ Взаимодействий между Пользователями</h2>
        <div class="chart-container">
            <canvas id="userInteractionsChart"></canvas>
        </div>
    
        <h2>Анализ Времени Отклика</h2>
        <div class="chart-container">
            <canvas id="responseTimesChart"></canvas>
        </div>
    
        <script>
            // Данные для графика активности по часам
            const activityByHourCtx = document.getElementById('activityByHourChart').getContext('2d');
            const activityByHourChart = new Chart(activityByHourCtx, {
                type: 'bar',
                data: {
                    labels: """ + json.dumps(hours) + """,
                    datasets: [{
                        label: 'Количество Сообщений',
                        data: """ + json.dumps(messages_per_hour) + """,
                        backgroundColor: 'rgba(54, 162, 235, 0.6)',
                        borderColor: 'rgba(54, 162, 235, 1)',
                        borderWidth: 1
                    }]
                },
                options: {
                    scales: {
                        x: {
                            title: {
                                display: true,
                                text: 'Часы'
                            }
                        },
                        y: {
                            beginAtZero: true,
                            title: {
                                display: true,
                                text: 'Количество Сообщений'
                            }
                        }
                    }
                }
            });
    
            // Данные для графика активности по дням недели
            const activityByWeekdayCtx = document.getElementById('activityByWeekdayChart').getContext('2d');
            const activityByWeekdayChart = new Chart(activityByWeekdayCtx, {
                type: 'bar',
                data: {
                    labels: """ + json.dumps(weekdays) + """,
                    datasets: [{
                        label: 'Количество Сообщений',
                        data: """ + json.dumps(messages_per_weekday) + """,
                        backgroundColor: 'rgba(255, 99, 132, 0.6)',
                        borderColor: 'rgba(255, 99, 132, 1)',
                        borderWidth: 1
                    }]
                },
                options: {
                    scales: {
                        x: {
                            title: {
                                display: true,
                                text: 'Дни Недели'
                            }
                        },
                        y: {
                            beginAtZero: true,
                            title: {
                                display: true,
                                text: 'Количество Сообщений'
                            }
                        }
                    }
                }
            });
    
            // Данные для графика активности по месяцам
            const activityByMonthCtx = document.getElementById('activityByMonthChart').getContext('2d');
            const activityByMonthChart = new Chart(activityByMonthCtx, {
                type: 'bar',
                data: {
                    labels: """ + json.dumps(months) + """,
                    datasets: [{
                        label: 'Количество Сообщений',
                        data: """ + json.dumps(messages_per_month) + """,
                        backgroundColor: 'rgba(75, 192, 192, 0.6)',
                        borderColor: 'rgba(75, 192, 192, 1)',
                        borderWidth: 1
                    }]
                },
                options: {
                    scales: {
                        x: {
                            title: {
                                display: true,
                                text: 'Месяцы'
                            }
                        },
                        y: {
                            beginAtZero: true,
                            title: {
                                display: true,
                                text: 'Количество Сообщений'
                            }
                        }
                    }
                }
            });
    
            // Данные для графика активности по времени суток
            const activityByTimeOfDayCtx = document.getElementById('activityByTimeOfDayChart').getContext('2d');
            const activityByTimeOfDayChart = new Chart(activityByTimeOfDayCtx, {
                type: 'bar',
                data: {
                    labels: """ + json.dumps(time_of_day) + """,
                    datasets: [{
                        label: 'Количество Сообщений',
                        data: """ + json.dumps(messages_per_time_of_day) + """,
                        backgroundColor: 'rgba(153, 102, 255, 0.6)',
                        borderColor: 'rgba(153, 102, 255, 1)',
                        borderWidth: 1
                    }]
                },
                options: {
                    scales: {
                        x: {
                            title: {
                                display: true,
                                text: 'Время Суток'
                            }
                        },
                        y: {
                            beginAtZero: true,
                            title: {
                                display: true,
                                text: 'Количество Сообщений'
                            }
                        }
                    }
                }
            });
    
            // Данные для графика взаимодействий между пользователями
            const userInteractionsCtx = document.getElementById('userInteractionsChart').getContext('2d');
            const userInteractionsChart = new Chart(userInteractionsCtx, {
                type: 'bar',
                data: {
                    labels: """ + json.dumps(list(data['user_interactions'].keys())) + """,
                    datasets: [{
                        label: 'Количество Взаимодействий',
                        data: """ + json.dumps([sum(interactions.values()) for interactions in data['user_interactions'].values()]) + """,
                        backgroundColor: 'rgba(255, 99, 132, 0.6)',
                        borderColor: 'rgba(255, 99, 132, 1)',
                        borderWidth: 1
                    }]
                },
                options: {
                    scales: {
                        x: {
                            title: {
                                display: true,
                                text: 'Пользователи'
                            }
                        },
                        y: {
                            title: {
                                display: true,
                                text: 'Количество Взаимодействий'
                            }
                        }
                    }
                }
            });
    
            // Данные для графика времени отклика
            const responseTimesCtx = document.getElementById('responseTimesChart').getContext('2d');
            const responseTimesChart = new Chart(responseTimesCtx, {
                type: 'bar',
                data: {
                    labels: """ + json.dumps(list(data['response_times'].keys())) + """,
                    datasets: [{
                        label: 'Среднее Время Отклика (секунды)',
                        data: """ + json.dumps([sum(times) / len(times) if times else 0 for times in data['response_times'].values()]) + """,
                        backgroundColor: 'rgba(75, 192, 192, 0.6)',
                        borderColor: 'rgba(75, 192, 192, 1)',
                        borderWidth: 1
                    }]
                },
                options: {
                    scales: {
                        x: {
                            title: {
                                display: true,
                                text: 'Пользователи'
                            }
                        },
                        y: {
                            title: {
                                display: true,
                                text: 'Среднее Время Отклика (секунды)'
                            }
                        }
                    }
                }
            });
    
            // Инициализация DataTables
            $(document).ready(function() {
                $('#userMessagesTable').DataTable({
                    "paging": false,
                    "searching": false,
                    "info": false
                });
                $('#userAverageLengthsTable').DataTable({
                    "paging": false,
                    "searching": false,
                    "info": false
                });
                $('#sentimentScoresTable').DataTable({
                    "paging": false,
                    "searching": false,
                    "info": false
                });
                $('#wordFrequencyTable').DataTable({
                    "paging": false,
                    "searching": false,
                    "info": false
                });
            });
        </script>
    
    </body>
    </html>
    """
    
    return html
    

    def escape_html(text): replacements = { "&": "&", "<": "<", ">": ">", '"': """, "'": "'" } return ''.join(replacements.get(c, c) for c in text)

    if name == "main": input_file = r"C:\Users\Admin\Downloads\AyuGram Desktop\ChatExport_2024-09-22\result.json" output_json = r"C:\Users\Admin\Downloads\AyuGram Desktop\ChatExport_2024-09-22\cleaned_result.json" output_html = r"C:\Users\Admin\Downloads\AyuGram Desktop\ChatExport_2024-09-22\chat_statistics.html"

    data = load_json(input_file)
    if data:
        analyzed_data = analyze_data(data)
        save_json(analyzed_data['cleaned_messages'], output_json)
        html_content = generate_html(analyzed_data)
        save_html(html_content, output_html)                     {исправь данный код и исправь ошибку или предложи ее решение (Traceback (most recent call last):
    

    File "C:\Users\Admin\source\repos\CleanJsonScript2\CleanJsonScript2\CleanJsonScript2.py", line 7, in from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer ModuleNotFoundError: No module named 'vaderSentiment' Для продолжения нажмите любую клавишу . . .)}

  2. Assistant

    The error you're encountering:

    ModuleNotFoundError: No module named 'vaderSentiment'
    

    occurs because the Python module vaderSentiment is not installed in your environment. The vaderSentiment library is used in your script for sentiment analysis. To resolve this issue, you need to install the vaderSentiment package.

    Solution:

    1. Install vaderSentiment Module

    You can install the vaderSentiment module using pip. Open your command prompt or terminal and run the following command:

    pip install vaderSentiment
    

    If you're using Python 3 and the above command doesn't work, try:

    pip3 install vaderSentiment
    

    Note: Make sure that you install the package in the same Python environment where you're running your script. If you're using a virtual environment, ensure it's activated before running the install command.

    2. Verify the Installation

    After installing, you can verify that the module is installed by running:

    pip show vaderSentiment
    

    This should display information about the vaderSentiment package if it's installed correctly.

    3. Re-run Your Script

    Now that the module is installed, try running your script again:

    python your_script_name.py
    

    Additional Suggestions:

    Handle Missing Modules Gracefully

    To make your script more robust, you can add a try-except block to handle the case where vaderSentiment is not installed and provide a user-friendly message:

    try:
        from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
    except ModuleNotFoundError:
        print("The 'vaderSentiment' module is not installed. Please install it using 'pip install vaderSentiment' and try again.")
        exit()
    

    Create a requirements.txt File

    It's a good practice to include a requirements.txt file with your script, listing all the dependencies. This makes it easier to set up the environment in the future or for others who might use your code.

    Here's how you can create one:

    1. Create the requirements.txt File

      In your project directory, create a file named requirements.txt and add the following content:

      vaderSentiment
      seaborn
      pandas
      

      Add any other modules that your script depends on (e.g., json, os, numpy if used, etc.).

    2. Install Dependencies from requirements.txt

      In the command prompt or terminal, run:

      pip install -r requirements.txt
      

    Ensure All Dependencies Are Installed

    Review your script for any other external modules that need to be installed and include them in your requirements.txt file. Modules like seaborn, pandas, and others also need to be installed separately if they're not part of the standard library.

    For example, to install seaborn and pandas:

    pip install seaborn pandas
    

    Summary:

    • Install the missing module using pip install vaderSentiment.
    • Consider adding error handling to check for required modules.
    • Use a requirements.txt file to manage dependencies.

    Next Steps:

    After installing the required modules, your script should run without the ModuleNotFoundError. If you encounter any other issues or have questions about the code, feel free to ask, and I'll be happy to assist further.