import pandas as pd
from neo4j import graphdatabase
from openai import openai
client = openai(api_key="")
product_data_df = pd.read_csv('../data/product_data.csv')
def preprocessing(df, columns_to_replace):
"""
used to preprocess certain column in dataframe
:param df: pandas dataframe - data
:param columns_to_replace: list - column name list
:return: df: pandas dataframe - processed data
"""
df[columns_to_replace] = df[columns_to_replace].apply(lambda col: col.str.replace("'s", "s"))
df[columns_to_replace] = df[columns_to_replace].apply(lambda col: col.str.replace("'", ""))
return df
def get_embedding(text):
"""
used to generate embeddings using openai embeddings model
:param text: str - text that needs to be converted to embeddings
:return: embedding
"""
model = "text-embedding-3-small"
text = text.replace("\n", " ")
return client.embeddings.create(input=[text], model=model).data[0].embedding
def create_category(product_data_df):
"""
used to generate queries for creating category nodes in neo4j
:param product_data_df: pandas dataframe - data
:return: query_list: list - list containing all create node queries for category
"""
cat_query = """create (a:category {name: '%s', embedding: %s})"""
distinct_category = product_data_df['category'].unique()
query_list = []
for category in distinct_category:
embedding = get_embedding(category)
query_list.append(cat_query % (category, embedding))
return query_list
def create_product(product_data_df):
"""
used to generate queries for creating product nodes in neo4j
:param product_data_df: pandas dataframe - data
:return: query_list: list - list containing all create node queries for product
"""
product_query = """create (a:product {name: '%s', description: '%s', price: %d, warranty_period: %d,
available_stock: %d, review_rating: %f, product_release_date: date('%s'), embedding: %s})"""
query_list = []
for idx, row in product_data_df.iterrows():
embedding = get_embedding(row['product name'] + " - " + row['description'])
query_list.append(product_query % (row['product name'], row['description'], int(row['price (inr)']),
int(row['warranty period (years)']), int(row['stock']),
float(row['review rating']), str(row['product release date']), embedding))
return query_list
def execute_bulk_query(query_list):
"""
executes queries is a list one by one
:param query_list: list - list of cypher queries
:return: none
"""
url = "bolt://localhost:7687"
auth = ("neo4j", "neo4j@123")
with graphdatabase.driver(url, auth=auth) as driver:
with driver.session() as session:
for query in query_list:
try:
session.run(query)
except exception as error:
print(f"error in executing query - {query}, error - {error}")
# preprocessing
product_data_df = preprocessing(product_data_df, ['product name', 'description'])
# create category
query_list = create_category(product_data_df)
execute_bulk_query(query_list)
# create product
query_list = create_product(product_data_df)
execute_bulk_query(query_list)
