import os import streamlit as st from streamlittags import sttags from datetime import…
- User
import os import streamlit as st from streamlit_tags import st_tags from datetime import datetime from io import BytesIO from scraper import perform_scrape from tools import html_to_markdown_with_readability import pandas as pd
Add custom CSS for table styling
CUSTOM_CSS = """
"""
def main(): st.markdown(CUSTOM_CSS, unsafe_allow_html=True)
# Main title with centered alignment st.markdown("<h1 style='text-align: center;'>Universal Web Scraper 🐙</h1>", unsafe_allow_html=True) tr=False # Sidebar for settings with st.sidebar: st.header("Web Scraper Settings") model = st.selectbox("Select Model", ["gemini flash-1.5"]) url = st.text_input("Enter URL") # Tag input for fields to extract fields_to_extract = st_tags( label="Fields to Extract", text="Press enter to add a tag", value=st.session_state.fields, suggestions=[], key="field_tags" ) # Prevent duplicate tags unique_fields = [] for field in fields_to_extract: if field not in unique_fields: unique_fields.append(field) st.session_state.fields = unique_fields tr=st.button("Scrape") if tr : if not url: st.error("Please enter a URL") st.stop() if not unique_fields: st.error("Please enter at least one field to extract") st.stop() try: # Assuming perform_scrape is a function that returns the required data df, input_tokens, output_tokens, total_cost = perform_scrape(url, unique_fields, model) timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') if len(df) > 0: st.success(f"Successfully extracted {len(df)} entries!") # Centered output table with custom styling st.markdown("<div style='display: flex; justify-content: center; flex-direction: column; align-items: center;'>", unsafe_allow_html=True) st.markdown("<h2 style='text-align: center;'>Extracted Data</h2>", unsafe_allow_html=True) st.markdown("<div class='table-container' style='width: 100%; max-width: 800px;'>", unsafe_allow_html=True) st.dataframe(df, use_container_width=True) st.markdown("</div>", unsafe_allow_html=True) # Centered download buttons st.markdown("<div style='display: flex; justify-content: center; gap: 1rem; width: 100%; max-width: 800px;'>", unsafe_allow_html=True) csv = df.to_csv(index=False) st.download_button( label="Download Data as CSV", data=csv, file_name=f"scraped_data_{timestamp}.csv", mime="text/csv" ) output = BytesIO() with pd.ExcelWriter(output, engine='xlsxwriter') as writer: df.to_excel(writer, index=False, sheet_name="ScrapedData") xlsx_data = output.getvalue() st.download_button( label="Download Data as XLSX", data=xlsx_data, file_name=f"scraped_data_{timestamp}.xlsx", mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ) markdown = df.to_markdown(index=False) st.download_button( label="Download Data as Markdown", data=markdown, file_name=f"scraped_data_{timestamp}.md", mime="text/markdown" ) json_data = df.to_json(orient="records") st.download_button( label="Download Data as JSON", data=json_data, file_name=f"scraped_data_{timestamp}.json", mime="application/json" ) st.markdown("</div>", unsafe_allow_html=True) st.markdown("</div>", unsafe_allow_html=True) with st.sidebar: col1, col2, col3 = st.sidebar.columns(3) with col1: st.metric("Input Tokens", input_tokens) with col2: st.metric("Output Tokens", output_tokens) with col3: st.metric("Total Tokens", input_tokens + output_tokens) except Exception as e: st.error(f"Error during scraping: {str(e)}")if name == "main": if 'fields' not in st.session_state: st.session_state.fields = [] main()
- Assistant
This Streamlit application is designed to be a universal web scraper. Here's a breakdown of its components and functionality:
-
Imports: The necessary libraries are imported:
os: Standard library for interacting with the operating system.streamlitasst: For building the web app UI.streamlit_tags: To allow users to dynamically add tags.datetime: For timestamping files.BytesIO: For handling byte streams.pandasaspd: For data manipulation and exporting.- Custom functions from
scraperandtoolsmodules.
-
Custom CSS: Adds styling to the data tables to control their appearance within the Streamlit app.
-
Main Functionality:
- Title: Displays a centered main title using HTML within Streamlit.
- Sidebar:
- A selectbox to choose the model (though only "gemini flash-1.5" is listed).
- A text input for the URL to scrape.
- A tag input section where users can specify fields they wish to extract.
- Data Handling:
- Ensures no duplicate tags by storing unique fields.
- Scrape button (
tr) triggers scraping when clicked.
- Scraping Logic:
- If a URL or fields are missing, displays an error.
- Calls
perform_scrapeto perform the actual scraping, which should return a DataFrame and some metrics. - If the DataFrame has data, displays success and shows the results in a styled table.
- Download Options:
- Provides buttons to download the scraped data in CSV, Excel, Markdown, and JSON formats.
- Metrics Display: Shows the number of input, output, and total tokens processed during scraping.
-
Error Handling: If any exceptions occur during scraping, an error message is displayed.
-
Session State: Uses
st.session_stateto persist the fields entered by the user across different sessions/interactions.
Note:
- The app assumes that
perform_scrapeandhtml_to_markdown_with_readabilityare implemented in external modules (scraperandtools). These need to handle the actual web scraping logic and conversions. - Ensure these modules are functional and correctly imported for the app to run as expected.
- Since web scraping often involves parsing HTML and potentially significant network interactions, it's crucial to handle exceptions and test with various URLs for robustness.
-
- User
need a better ui than this
- Assistant
Enhancing the UI of your Streamlit application can significantly improve user experience. Here are several suggestions to improve the aesthetics and functionality of the current application:
1. Use Streamlit's Built-in Themes
Streamlit provides dark and light mode themes. You can customize the appearance using these built-in themes. To set a theme, create a configuration file
~/.streamlit/config.tomlwith your desired settings:[theme] primaryColor = "#4A4A4A" backgroundColor = "#F1F1F1" secondaryBackgroundColor = "#E0E0E0" textColor = "#333333" font = "sans serif"2. Leverage Columns and Expander
Use columns to organize content side-by-side and employ expanders for less frequently accessed information:
import streamlit as st st.set_page_config(layout="wide") def main(): st.title("Universal Web Scraper 🐙") # Data input section with st.sidebar: st.header("Settings") model = st.selectbox("Select Model", ["gemini flash-1.5"]) url = st.text_input("Enter URL") with st.expander("Advanced Settings"): fields_to_extract = st_tags( label="Fields to Extract", text="Press enter to add a tag", value=st.session_state.fields if 'fields' in st.session_state else [], suggestions=[], key="field_tags" ) scrape_button = st.button("Scrape") # Main section if scrape_button: # Add your scraping logic here pass if __name__ == "__main__": main()3. Improve Visual Hierarchy
Use headers and color to distinguish different parts of your application clearly. Adding icons or images can make the app more visually appealing.
st.markdown('<h1 style="text-align: center; color: #4A4A4A;">Web Scraper Dashboard</h1>', unsafe_allow_html=True)4. Interactive Elements and Feedback
- Use progress bars or spinner widgets to indicate activity during scraping operations.
- Use
st.toastwhen available to give users instant feedback on actions, such as successful scrape completion.
import time if scrape_button: with st.spinner('Scraping data...'): time.sleep(2) # Simulating a delay st.success("Scraping complete!")5. Improve Data Presentation
- Use Streamlit's
st.dataframeorst.tablefor better dataset displays. - Consider plotting statistics or results using Streamlit's chart functions for visual insights.
6. Better Inputs and Controls
Replace the text input for URLs with file uploads, or add a multi-input widget if users typically enter multiple URLs.
7. Incorporate Markdown and Rich Content
Spruce up the interface using Streamlit's ability to interpret HTML/Markdown for custom stylings.
st.markdown("### Results") if dataframe: st.dataframe(dataframe.style.highlight_max(axis=0))Sample UI Structure
Here's a sample UI structure based on these improvements:
import streamlit as st from datetime import datetime st.set_page_config(page_title="Web Scraper", layout="wide") def main(): st.title("🕸️ Universal Web Scraper") with st.sidebar: st.header("Configurations") model = st.selectbox("Model", options=["gemini flash-1.5"], index=0) st.write("Configure your web scraping preferences.") url = st.text_input("Target URL", placeholder="e.g., https://example.com") st.subheader("Fields to Extract") fields = st.text_area("Enter comma-separated fields", placeholder="e.g., title, date, author") scrape_button = st.button("🕵️♂️ Scrape Now") if scrape_button and url: st.subheader("Scraping Progress") with st.spinner("Working on it..."): # Simulate data scraping logic time.sleep(2) st.success("Data Scraped Successfully!") st.subheader("Results") df = st.experimental_data_editor(df, num_rows="dynamic") else: st.warning("Please ensure well-formed input.") if __name__ == "__main__": main()Final Note
Remember, always validate and handle user inputs properly to ensure the app's responsiveness, robustness, and consistency. Also, consider adding tutorial-like descriptions within the app for better user guidance.
- User
need complete ui code with updated improvements
- Assistant
Certainly! Here's an updated version of the Streamlit web scraper app with improved UI and layout, using the suggestions provided earlier. This example assumes you're still importing the key functions you'd need for actual scraping (
perform_scrapeand others you might need to define):import streamlit as st from streamlit_tags import st_tags import pandas as pd from datetime import datetime from io import BytesIO from scraper import perform_scrape # Set the page configuration st.set_page_config(page_title="Universal Web Scraper", layout="wide") # Custom CSS for styling the table and other elements CUSTOM_CSS = """ <style> .table-container { display: flex; justify-content: center; margin: 2rem auto; max-width: 80%; } .stDataframe { width: 100%; } .dataframe { margin: 0 auto; } </style> """ # Main function def main(): st.markdown(CUSTOM_CSS, unsafe_allow_html=True) # Title with enhanced styling st.markdown("<h1 style='text-align: center; color: #4A4A4A;'>Universal Web Scraper 🐙</h1>", unsafe_allow_html=True) # Sidebar configuration section with st.sidebar: st.header("Configure Your Scraper") # Model selection (example list) model = st.selectbox("Choose Model", ["gemini flash-1.5"]) # URL input url = st.text_input("Enter URL", placeholder="e.g., https://example.com") # Fields to extract using tags fields_to_extract = st_tags( label="Fields to Extract", text="Press Enter to Add a Field", value=st.session_state.fields if 'fields' in st.session_state else [], suggestions=[], key="field_tags" ) # Ensure no duplicate fields unique_fields = list(set(fields_to_extract)) st.session_state.fields = unique_fields # Scrape button scrape_button = st.button("Scrape") if scrape_button: if not url: st.error("Please enter a URL.") st.stop() if not unique_fields: st.error("Please enter at least one field to extract.") st.stop() try: # Invoke the scraper function (assuming this is defined in your scraper module) df, input_tokens, output_tokens, total_cost = perform_scrape(url, unique_fields, model) if not df.empty: st.success(f"Successfully extracted {len(df)} entries!") # Display the results in an interactive dataframe st.subheader("Extracted Data") st.dataframe(df, use_container_width=True) # Prepare downloads st.markdown("### Download Options") timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') # CSV csv = df.to_csv(index=False).encode('utf-8') st.download_button( label="Download as CSV", data=csv, file_name=f"scraped_data_{timestamp}.csv", mime="text/csv" ) # Excel output = BytesIO() with pd.ExcelWriter(output, engine='xlsxwriter') as writer: df.to_excel(writer, index=False, sheet_name="ScrapedData") xlsx_data = output.getvalue() st.download_button( label="Download as Excel", data=xlsx_data, file_name=f"scraped_data_{timestamp}.xlsx", mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ) # JSON json_data = df.to_json(orient="records") st.download_button( label="Download as JSON", data=json_data, file_name=f"scraped_data_{timestamp}.json", mime="application/json" ) # Markdown markdown = df.to_markdown(index=False) st.download_button( label="Download as Markdown", data=markdown, file_name=f"scraped_data_{timestamp}.md", mime="text/markdown" ) # Sidebar metrics with st.sidebar: st.subheader("Token Metrics") col1, col2, col3 = st.columns(3) col1.metric("Input Tokens", input_tokens) col2.metric("Output Tokens", output_tokens) col3.metric("Total Tokens", input_tokens + output_tokens) else: st.warning("No data extracted. Check the URL and fields.") except Exception as e: st.error(f"An error occurred during scraping: {str(e)}") if __name__ == "__main__": if 'fields' not in st.session_state: st.session_state.fields = [] main()Key Features:
- Responsive Layout: Streamlit's wide layout is employed to make the best use of screen space.
- Enhanced Styling: The title and other text elements are styled for better appearance, and the table container is centrally aligned.
- User Feedback: Multiple download options are provided, and token metrics are displayed to give users insights into the scraping process.
- Interactivity: Scraper input fields use tag systems for easy management, and buttons provide clear actions.
- Exception Handling: Provides clear feedback in case of errors or if scraping yields no data.
Note:
Ensure your
perform_scrapefunction is properly implemented to return the required data and metrics from the scraping process. Also, modify the example to suit your specific scraping needs and data models.