ingest.py

import os
import streamlit as st
import json
from langchain.schema import AIMessage
from langchain.adapters.openai import convert_message_to_dict
import time
import openai
from io import BytesIO
import pandas as pd

os.environ['OPENAI_API_KEY'] = st.secrets['OPENAI_API_KEY']



if __name__ == "__main__":
    d = pd.read_csv('political_social_media.csv', encoding_errors= "ignore")
    result = d.reset_index().to_json(orient="records")
    data = json.loads(result)
    # for key, value in data.items():
    #     print(key, value)

    tweets = [d["text"] for d in data]
    messages = [AIMessage(content=t) for t in tweets]
    system_message = {"role": "system", "content": "write a tweet"}
    data = [[system_message, convert_message_to_dict(m)] for m in messages]
    # print(data)

    my_file = BytesIO()
    for m in data:
        my_file.write((json.dumps({"messages": m}) + "\n").encode('utf-8'))

    my_file.seek(0)
    training_file = openai.File.create(
      file=my_file,
      purpose='fine-tune'
    )
    # while True:
    #     try:
    #         job = openai.FineTuningJob.create(training_file=training_file.id, model="gpt-3.5-turbo")
    #     except Exception as e:
    #         print(e)
    #         print("Trying again in ten seconds....")
    #         time.sleep(10)

    start = time.time()

    while True:
        ftj = openai.FineTuningJob.retrieve("ftjob-fVHiKTzMwuqei6v2AyEtjygd")
        if ftj.fine_tuned_model is None:
            print(f"Waiting for fine-tuning to complete... Elapsed: {time.time() - start}", end="\r", flush=True)
            time.sleep(10)
        else:
            print("\n")
            print(ftj.fine_tuned_model, flush=True)
            break