if session.view is not None: classificationviews = [] for sample in session.view:…
- User
if session.view is not None: classification_views = []
for sample in session.view: first_frame = sample.frames.first() if first_frame.classifications and first_frame.classifications.classifications: skip_full_frame_catalog = False for classification in first_frame.classifications.classifications: start, stop, step = 0, sample.metadata.total_frame_count, 1 if classification.label == "video_split": start, stop = classification.start, classification.stop elif classification.label == "single_frames": start, stop, step = classification.start, classification.stop, classification.step else: continue frame_ids_for_classification = [ frame.id for frame in sample.frames.values() if start <= frame.frame_number <= stop and (frame.frame_number - start) % step == 0 ] if frame_ids_for_classification: classification_view = session.view.select_frames(frame_ids_for_classification) classification_views.append(classification_view) skip_full_frame_catalog = True if skip_full_frame_catalog: continue start, stop, step = 0, sample.metadata.total_frame_count, 1 frame_ids_for_default = [ frame.id for frame in sample.frames.values() if start <= frame.frame_number <= stop and (frame.frame_number - start) % step == 0 ] if frame_ids_for_default: default_view = session.view.select_frames(frame_ids_for_default) classification_views.append(default_view) for view in classification_views: first_frame = view.first() parts = first_frame.filepath.split('/') og_data_loc = os.path.join(minio_endpoint, *parts[3:]) frames_view = view.to_frames(sample_frames=True, sparse=True) first_frame = frames_view.first() parts = first_frame.filepath.split('/') bucket_name = parts[3] rest_of_path = '/'.join(parts[4:-1]) now = datetime.datetime.now() date_time_folder = now.strftime("%Y%m%d_%H%M%S") folder_path = f"fiftyone_processed_data/{rest_of_path}/{date_time_folder}" local_video_directory = os.path.join("/".join(parts[:4]), os.path.dirname(rest_of_path)) video_file_name = f"stitched_{date_time_folder}.mp4" local_video_path = os.path.join(local_video_directory, video_file_name) # Determine the video format from the first frame extension = os.path.splitext(og_data_loc)[-1].upper() data_type = extension[1:] # Remove the dot from the extension # Initialize video writer frame = cv2.imread(first_frame.filepath) height, width, layers = frame.shape codec = cv2.VideoWriter_fourcc(*'mp4v') video = cv2.VideoWriter(local_video_path, codec, view.first().metadata.frame_rate, (width, height)) for frame in frames_view: frame_path = frame.filepath frame = cv2.imread(frame_path) video.write(frame) video.release() s3_upload_path = os.path.join(folder_path, video_file_name) try: with open(local_video_path, "rb") as f: s3_client.upload_fileobj(f, bucket_name, s3_upload_path) except Exception as e: print(f"Failed to upload stitched video due to {e}") task_name = video_file_name with ApiClient(configuration) as api_client: try: twr = TaskWriteRequest(task_name, project_id=target_project) data, response = api_client.tasks_api.create(twr, org=organization) server_files = [f"{bucket_name}/{s3_upload_path}"] data_request = DataRequest(image_quality=95, server_files=server_files, use_cache=True, use_zip_chunks=False) api_client.tasks_api.create_data(data.id, upload_finish=True, upload_multiple=False, upload_start=True, data_request=data_request) print("Uploaded task to CVAT") except exceptions.ApiException as e: print(f"Exception when calling CVAT API: {e}")The goal of this code is, given a project A, cataloging project, upload to project B with a downselect of the data. This works as expected. I have realized if the user isn't splitting the data, and going to: if skip_full_frame_catalog: area, it will essentially be the unmodified file. Which means we can use the original source and do not need to s3_client.upload_fileobj(f, bucket_name, s3_upload_path). Do you understand? If so, go ahead an fix that so we prevent data duplication where possible. Give me the entire code back, no omissions.
- Assistant
Yes, I understand the need to optimize the process to avoid data duplication for unmodified files by utilizing the original source instead of uploading the stitched videos again. Here's the revised code with the necessary modifications:
import cv2 import datetime import os from fiftyone.core.session import Session from some_api_client import ApiClient, exceptions from some_api_client.models import TaskWriteRequest, DataRequest # Assuming these necessary variables are defined: # session, os, datetime, cv2, s3_client, minio_endpoint, configuration, target_project, organization if session.view is not None: classification_views = [] for sample in session.view: first_frame = sample.frames.first() if first_frame.classifications and first_frame.classifications.classifications: skip_full_frame_catalog = False for classification in first_frame.classifications.classifications: start, stop, step = 0, sample.metadata.total_frame_count, 1 if classification.label == "video_split": start, stop = classification.start, classification.stop elif classification.label == "single_frames": start, stop, step = classification.start, classification.stop, classification.step else: continue frame_ids_for_classification = [ frame.id for frame in sample.frames.values() if start <= frame.frame_number <= stop and (frame.frame_number - start) % step == 0 ] if frame_ids_for_classification: classification_view = session.view.select_frames(frame_ids_for_classification) classification_views.append(classification_view) skip_full_frame_catalog = True if skip_full_frame_catalog: continue start, stop, step = 0, sample.metadata.total_frame_count, 1 frame_ids_for_default = [ frame.id for frame in sample.frames.values() if start <= frame.frame_number <= stop and (frame.frame_number - start) % step == 0 ] if frame_ids_for_default: default_view = session.view.select_frames(frame_ids_for_default) classification_views.append(default_view) for view in classification_views: frames_view = view.to_frames(sample_frames=True, sparse=True) use_original_source = len(frames_view) == view.first().metadata.total_frame_count if use_original_source: og_data_loc = view.first().filepath # If using the original source, set local_video_path to og_data_loc local_video_path = og_data_loc else: first_frame = frames_view.first() parts = first_frame.filepath.split('/') local_video_directory = os.path.join("/".join(parts[:4]), os.path.dirname('/'.join(parts[4:-1]))) now = datetime.datetime.now() date_time_folder = now.strftime("%Y%m%d_%H%M%S") video_file_name = f"stitched_{date_time_folder}.mp4" local_video_path = os.path.join(local_video_directory, video_file_name) # Initialize video writer frame = cv2.imread(first_frame.filepath) height, width, layers = frame.shape codec = cv2.VideoWriter_fourcc(*'mp4v') video = cv2.VideoWriter(local_video_path, codec, view.first().metadata.frame_rate, (width, height)) for frame in frames_view: frame_path = frame.filepath frame = cv2.imread(frame_path) video.write(frame) video.release() bucket_name = parts[3] rest_of_path = '/'.join(parts[4:-1]) folder_path = f"fiftyone_processed_data/{rest_of_path}/{date_time_folder}" s3_upload_path = os.path.join(folder_path, video_file_name) if not use_original_source: try: with open(local_video_path, "rb") as f: s3_client.upload_fileobj(f, bucket_name, s3_upload_path) except Exception as e: print(f"Failed to upload stitched video due to {e}") task_name = video_file_name with ApiClient(configuration) as api_client: try: twr = TaskWriteRequest(task_name, project_id=target_project) data, response = api_client.tasks_api.create(twr, org=organization) server_files = [f"{bucket_name}/{s3_upload_path}" if not use_original_source else view.first().metadata['source_path']] data_request = DataRequest(image_quality=95, server_files=server_files, use_cache=True, use_zip_chunks=False) api_client.tasks_api.create_data(data.id, upload_finish=True, upload_multiple=False, upload_start=True, data_request=data_request) print("Uploaded task to CVAT") except exceptions.ApiException as e: print(f"Exception when calling CVAT API: {e}")This modification checks if the operation needs to process and upload a stitched video or if it could directly use the original source to avoid unnecessary data duplication. The change leverages a flag,
use_original_source, to decide the workflow path. The S3 upload and CVAT API upload sections are updated accordingly to reflect this choice.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.