-- coding: utf-8 -- """notebook.ipynb Automatically generated by Colab. Original file is…
- User
-- coding: utf-8 --
"""notebook.ipynb
Automatically generated by Colab.
Original file is located at https://colab.research.google.com/drive/10KllEMzGTWHKwkZLMq9YFnn7eLO6Y6Oy
Latihan Membuat Machine Learning Pipeline
"""
!pip install tfx
import tensorflow as tf from tfx.components import CsvExampleGen, StatisticsGen, SchemaGen, ExampleValidator, Transform, Trainer, Tuner from tfx.proto import example_gen_pb2 from tfx.orchestration.experimental.interactive.interactive_context import InteractiveContext import os
Commented out IPython magic to ensure Python compatibility.
from google.colab import drive drive.mount('/content/drive')
%cd /content/drive/MyDrive/Pengumpulan Tugas/MLOps ML Pipeline
"""## Set Variable"""
PIPELINE_NAME = "sarcasm-pipeline" SCHEMA_PIPELINE_NAME = "sarcasm-tfdv-schema"
#Directory untuk menyimpan artifact yang akan dihasilkan PIPELINE_ROOT = os.path.join('pipelines', PIPELINE_NAME)
Path to a SQLite DB file to use as an MLMD storage.
METADATA_PATH = os.path.join('metadata', PIPELINE_NAME, 'metadata.db')
Output directory where created models from the pipeline will be exported.
SERVING_MODEL_DIR = os.path.join('serving_model', PIPELINE_NAME)
from absl import logging
logging.set_verbosity(logging.INFO)
DATA_ROOT = "data"
interactive_context = InteractiveContext(pipeline_root=PIPELINE_ROOT)
"""## Pembuatan Komponen Pengolahan Data"""
output = example_gen_pb2.Output( split_config = example_gen_pb2.SplitConfig(splits=[ example_gen_pb2.SplitConfig.Split(name="train", hash_buckets=8), example_gen_pb2.SplitConfig.Split(name="eval", hash_buckets=2) ]) ) example_gen = CsvExampleGen(input_base=DATA_ROOT, output_config=output) interactive_context.run(example_gen)
statistics_gen = StatisticsGen( examples=example_gen.outputs["examples"] ) interactive_context.run(statistics_gen)
interactive_context.show(statistics_gen.outputs["statistics"])
schema_gen = SchemaGen(statistics=statistics_gen.outputs["statistics"]) interactive_context.run(schema_gen)
interactive_context.show(schema_gen.outputs["schema"])
example_validator = ExampleValidator( statistics=statistics_gen.outputs['statistics'], schema=schema_gen.outputs['schema'] ) interactive_context.run(example_validator)
interactive_context.show(example_validator.outputs['anomalies'])
TRANSFORM_MODULE_FILE = "sarcasm_transform.py"
Commented out IPython magic to ensure Python compatibility.
%%writefile {TRANSFORM_MODULE_FILE}
import tensorflow as tf
LABEL_KEY = "is_sarcastic"
FEATURE_KEY = "headline"
def transformed_name(key):
"""Renaming transformed features"""
return key + "_xf"
def preprocessing_fn(inputs):
"""
Preprocess input features into transformed features
Args:
inputs: map from feature keys to raw features.
Return:
outputs: map from feature keys to transformed features.
"""
outputs = {}
outputs[transformed_name(FEATURE_KEY)] = tf.strings.lower(inputs[FEATURE_KEY])
outputs[transformed_name(LABEL_KEY)] = tf.cast(inputs[LABEL_KEY], tf.int64)
return outputs
transform = Transform( examples=example_gen.outputs['examples'], schema= schema_gen.outputs['schema'], module_file=os.path.abspath(TRANSFORM_MODULE_FILE) ) interactive_context.run(transform) buatlah,dokumen,rpoyek,berdasarkan,code,ini
- Assistant
To document this project, we'll create a structured document that includes an overview, key concepts, details of the implementation, and its outcomes. This kind of documentation is critical in machine learning and data science projects to help others understand and replicate your work.
Project Documentation: Sarcasm Detection Pipeline using TFX
Overview
This project demonstrates the creation of a Machine Learning pipeline for sarcasm detection using TensorFlow Extended (TFX). The pipeline processes data, validates it, and applies transformations in preparation for modeling. The main aim is to illustrate the steps needed to construct a robust ML pipeline with TFX components.
Project Components and Their Functions
-
Data Ingestion: Data is ingested using
CsvExampleGen, which reads data from CSV files and divides it into training and evaluation datasets through specified hash buckets. -
Statistics Generation: Using
StatisticsGen, the pipeline generates descriptive statistics about the dataset, which helps in understanding the data distribution and identifying potential issues. -
Schema Generation:
SchemaGenautomatically infers a schema, which is a detailed description of the dataset's feature types and expected properties. -
Data Validation:
ExampleValidatorvalidates the dataset against the schema to identify anomalies or outliers, ensuring clean and consistent data input for the model. -
Data Transformation: In
Transform, data preprocessing steps are defined. This project focuses on simple text transformation such as changing headlines to lowercase, which is crucial for text processing.
Implementation Details
Prerequisites
- Google Colab Environment: The project is designed to run in a Google Colab environment.
- TFX Installation: TensorFlow Extended is used for building the pipeline, which requires installation using
pip install tfx.
Directory Setup
PIPELINE_ROOT: Directory for storing pipeline artifacts.METADATA_PATH: Path to the SQLite database file for logging metadata.SERVING_MODEL_DIR: Directory for exporting the trained model.DATA_ROOT: Directory containing the dataset.
Key Script Sections
Mounting Google Drive
from google.colab import drive drive.mount('/content/drive')Ingesting and Splitting Data
example_gen = CsvExampleGen( input_base=DATA_ROOT, output_config=output) interactive_context.run(example_gen)Statistics and Schema Generation/Validation
statistics_gen = StatisticsGen( examples=example_gen.outputs["examples"] ) interactive_context.run(statistics_gen) schema_gen = SchemaGen(statistics=statistics_gen.outputs["statistics"]) interactive_context.run(schema_gen)Detecting Anomalies
example_validator = ExampleValidator( statistics=statistics_gen.outputs['statistics'], schema=schema_gen.outputs['schema'] ) interactive_context.run(example_validator)Data Transformation
TRANSFORM_MODULE_FILE = "sarcasm_transform.py" transform = Transform( examples=example_gen.outputs['examples'], schema= schema_gen.outputs['schema'], module_file=os.path.abspath(TRANSFORM_MODULE_FILE) ) interactive_context.run(transform)Transform Module
The transformation module, defined in
sarcasm_transform.py, lowers the case of headlines and converts the label to an integer:import tensorflow as tf LABEL_KEY = "is_sarcastic" FEATURE_KEY = "headline" def transformed_name(key): return key + "_xf" def preprocessing_fn(inputs): outputs = {} outputs[transformed_name(FEATURE_KEY)] = tf.strings.lower(inputs[FEATURE_KEY]) outputs[transformed_name(LABEL_KEY)] = tf.cast(inputs[LABEL_KEY], tf.int64) return outputsOutcomes and Discussion
- The pipeline successfully ingests data, generates statistics, infers a schema, validates the data, and applies transformations.
- Handling and pre-processing text data such as changing text to lowercase is crucial for text-based machine learning tasks.
- The modular approach helps in maintaining the code, making it easy to adjust components without altering the entire pipeline.
Future Work
- Implementing a model training and evaluation step once data is pre-processed.
- Expanding transformation logic to include more sophisticated NLP techniques.
- Deploying the trained model for real-time sarcasm detection.
This documentation should serve as a comprehensive guide to the project's structure and purpose, allowing other developers or students to understand and replicate the process in similar ML pipeline projects.
-
- User
dalam,b,indo
- Assistant
Berikut adalah dokumentasi proyek dalam Bahasa Indonesia:
Dokumentasi Proyek: Pipeline Deteksi Sarkasme Menggunakan TFX
Ringkasan
Proyek ini mendemonstrasikan pembuatan pipeline Machine Learning untuk deteksi sarkasme menggunakan TensorFlow Extended (TFX). Pipeline ini memproses data, memvalidasi, dan menerapkan transformasi untuk mempersiapkan data sebelum pemodelan. Tujuan utama dari proyek ini adalah untuk menggambarkan langkah-langkah yang diperlukan dalam membangun pipeline ML yang kuat dengan komponen TFX.
Komponen Proyek dan Fungsinya
-
Ingesti Data: Data dimasukkan menggunakan
CsvExampleGen, yang membaca data dari file CSV dan membaginya menjadi dataset pelatihan dan evaluasi melalui hash buckets yang ditentukan. -
Generasi Statistik: Dengan
StatisticsGen, pipeline menghasilkan statistik deskriptif tentang dataset yang membantu memahami distribusi data dan mengidentifikasi masalah potensial. -
Generasi Skema:
SchemaGensecara otomatis menginfeksi skema, yaitu deskripsi rinci tentang tipe fitur dan properti yang diharapkan dalam dataset. -
Validasi Data:
ExampleValidatormemvalidasi dataset terhadap skema untuk mengidentifikasi anomali atau data yang menyimpang, memastikan data yang masuk ke model bersih dan konsisten. -
Transformasi Data: Dalam
Transform, langkah-langkah pra-pemrosesan data didefinisikan. Proyek ini berfokus pada transformasi teks sederhana seperti mengubah judul menjadi huruf kecil, yang penting untuk pemrosesan teks.
Detail Implementasi
Prasyarat
- Lingkungan Google Colab: Proyek ini dirancang untuk berjalan di lingkungan Google Colab.
- Instalasi TFX: TensorFlow Extended digunakan untuk membangun pipeline, yang memerlukan instalasi dengan
pip install tfx.
Pengaturan Direktori
PIPELINE_ROOT: Direktori untuk menyimpan artefak pipeline.METADATA_PATH: Jalur ke file database SQLite untuk mencatat metadata.SERVING_MODEL_DIR: Direktori untuk mengekspor model yang telah dilatih.DATA_ROOT: Direktori yang berisi dataset.
Bagian Kunci dari Skrip
Mounting Google Drive
from google.colab import drive drive.mount('/content/drive')Ingesti dan Pembagian Data
example_gen = CsvExampleGen( input_base=DATA_ROOT, output_config=output) interactive_context.run(example_gen)Generasi Statistik dan Skema/Validasi
statistics_gen = StatisticsGen( examples=example_gen.outputs["examples"] ) interactive_context.run(statistics_gen) schema_gen = SchemaGen(statistics=statistics_gen.outputs["statistics"]) interactive_context.run(schema_gen)Deteksi Anomali
example_validator = ExampleValidator( statistics=statistics_gen.outputs['statistics'], schema=schema_gen.outputs['schema'] ) interactive_context.run(example_validator)Transformasi Data
TRANSFORM_MODULE_FILE = "sarcasm_transform.py" transform = Transform( examples=example_gen.outputs['examples'], schema= schema_gen.outputs['schema'], module_file=os.path.abspath(TRANSFORM_MODULE_FILE) ) interactive_context.run(transform)Modul Transformasi
Modul transformasi yang didefinisikan dalam
sarcasm_transform.pymenurunkan kasus headlines dan mengonversi label menjadi integer:import tensorflow as tf LABEL_KEY = "is_sarcastic" FEATURE_KEY = "headline" def transformed_name(key): return key + "_xf" def preprocessing_fn(inputs): outputs = {} outputs[transformed_name(FEATURE_KEY)] = tf.strings.lower(inputs[FEATURE_KEY]) outputs[transformed_name(LABEL_KEY)] = tf.cast(inputs[LABEL_KEY], tf.int64) return outputsHasil dan Diskusi
- Pipeline berhasil melakukan ingest, menghasilkan statistik, menginfeksi skema, memvalidasi data, dan menerapkan transformasi.
- Penanganan dan pra-pemrosesan teks seperti mengubah teks menjadi huruf kecil adalah hal penting dalam tugas pembelajaran mesin berbasis teks.
- Pendekatan modular membantu dalam memelihara kode, sehingga memudahkan penyesuaian komponen tanpa mengubah seluruh pipeline.
Pekerjaan Masa Depan
- Menerapkan langkah pelatihan dan evaluasi model setelah data diproses.
- Memperluas logika transformasi untuk menyertakan teknik NLP yang lebih canggih.
- Mendeploy model terlatih untuk deteksi sarkasme secara real-time.
Dokumentasi ini diharapkan dapat menjadi panduan komprehensif mengenai struktur dan tujuan proyek, sehingga memungkinkan pengembang lain atau siswa untuk memahami dan mereplikasikan proses dalam proyek pipeline ML serupa.
-