write some code with pyiceberg that writes file and creates simple http server to share…
- User
write some code with pyiceberg that writes file and creates simple http server to share the file
Assistant
Here's code that uses PyIceberg to create an Iceberg table, write data to it, and then create a simple HTTP server to share the files. However, I should note that this code won't work in the Pyodide console since Pyodide runs in a web browser and cannot create HTTP servers or access the local filesystem in the traditional way.
import os import tempfile from http.server import HTTPServer, SimpleHTTPRequestHandler import threading import pyarrow as pa from pyiceberg import catalog from pyiceberg.catalog import load_catalog from pyiceberg.schema import Schema from pyiceberg.types import NestedField, StringType, IntegerType, LongType from pyiceberg.partitioning import PartitionSpec, PartitionField from pyiceberg.transforms import IdentityTransform # Create a temporary directory for our Iceberg warehouse warehouse_path = tempfile.mkdtemp(prefix="iceberg_warehouse_") print(f"Warehouse path: {warehouse_path}") # Configuration for a local file-based catalog catalog_config = { "type": "sql", "uri": f"sqlite:///{warehouse_path}/catalog.db", "warehouse": f"file://{warehouse_path}" } # Create catalog iceberg_catalog = load_catalog("local", **catalog_config) # Define schema for our table schema = Schema( NestedField(1, "id", LongType(), required=True), NestedField(2, "name", StringType(), required=True), NestedField(3, "age", IntegerType(), required=True), NestedField(4, "city", StringType(), required=False) ) # Create a partitioned table partition_spec = PartitionSpec( PartitionField(4, 1000, IdentityTransform(), "city") ) # Create namespace and table namespace = "examples" table_name = "people" try: iceberg_catalog.create_namespace(namespace) except: pass # Namespace might already exist # Create table table = iceberg_catalog.create_table( f"{namespace}.{table_name}", schema=schema, partition_spec=partition_spec ) # Create sample data data = { "id": [1, 2, 3, 4, 5], "name": ["Alice", "Bob", "Charlie", "David", "Eve"], "age": [25, 30, 35, 28, 32], "city": ["NYC", "NYC", "LA", "Chicago", "LA"] } # Convert to PyArrow table arrow_table = pa.table(data) # Write data to the Iceberg table table.append(arrow_table) # Query the table to verify data scan = table.scan() result_df = scan.to_pandas() print("\nData in Iceberg table:") print(result_df) # Create a simple HTTP server to serve the warehouse directory class CustomHTTPRequestHandler(SimpleHTTPRequestHandler): def __init__(self, *args, **kwargs): super().__init__(*args, directory=warehouse_path, **kwargs) def end_headers(self): # Add CORS headers to allow access from web browsers self.send_header('Access-Control-Allow-Origin', '*') self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS') self.send_header('Access-Control-Allow-Headers', 'Content-Type') super().end_headers() def start_server(port=8000): """Start HTTP server in a separate thread""" server = HTTPServer(('localhost', port), CustomHTTPRequestHandler) print(f"\nHTTP Server started at http://localhost:{port}") print(f"Serving files from: {warehouse_path}") print("Press Ctrl+C to stop the server") # List some key files that can be accessed print("\nKey files you can access:") print(f"- Catalog DB: http://localhost:{port}/catalog.db") print(f"- Metadata: http://localhost:{port}/{namespace}/{table_name}/metadata/") server_thread = threading.Thread(target=server.serve_forever) server_thread.daemon = True server_thread.start() return server # Start the HTTP server server = start_server(8000) # Keep the main thread alive try: input("\nPress Enter to stop the server and clean up...") except KeyboardInterrupt: pass # Cleanup print(f"\nCleaning up temporary warehouse at: {warehouse_path}")For Pyodide Environment:
Since Pyodide runs in a web browser, you cannot create traditional HTTP servers or access the local filesystem. Instead, you would need to:
- Use in-memory storage or browser storage APIs (like IndexedDB)
- Use the File API to create downloadable files
- Use service workers for more advanced scenarios
Here's a modified version that might work better in Pyodide:
import io import json import pyarrow as pa import pyarrow.parquet as pq # Create sample data data = { "id": [1, 2, 3, 4, 5], "name": ["Alice", "Bob", "Charlie", "David", "Eve"], "age": [25, 30, 35, 28, 32], "city": ["NYC", "NYC", "LA", "Chicago", "LA"] } # Convert to PyArrow table arrow_table = pa.table(data) # Write to parquet in memory buffer = io.BytesIO() pq.write_table(arrow_table, buffer) buffer.seek(0) # In Pyodide, you can create a downloadable link parquet_bytes = buffer.getvalue() # Convert to base64 for creating a data URL import base64 parquet_base64 = base64.b64encode(parquet_bytes).decode('utf-8') # Create a data URL data_url = f"data:application/octet-stream;base64,{parquet_base64}" # In Pyodide, you would use JavaScript to create a downloadable link print(f"Data URL created. Length: {len(data_url)} characters") print("To download in browser, you would need to create an HTML anchor element with this data URL") # Also create a JSON representation for easier viewing json_data = arrow_table.to_pandas().to_json(orient='records') print(f"\nJSON representation of the data:") print(json_data)Note that full PyIceberg functionality with catalogs and metadata management is likely not available in Pyodide due to its filesystem and networking limitations. You would primarily be limited to working with in-memory data structures.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.