LLM Sentence Embedding向量化相似性搜索技术初探

https://www.infoq.cn/article/j3vvye0uhlfdpnc8uyrs
https://36kr.com/p/2233027665457281
https://www.pinecone.io/learn/vector-database/
https://www.pinecone.io/

pip install chromadb
pip install pybind11
pip install tiktoken
pip install unstructured
pip install pdf2image
pip install pytesseract

import os
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.vectorstores import Chroma
from langchain.text_splitter import CharacterTextSplitter
from langchain import OpenAI, VectorDBQA
from langchain.document_loaders import DirectoryLoader
from langchain.chains import RetrievalQA


# 加载文件夹中的所有.neat文件
loader = DirectoryLoader('./webshell_data_0414', glob='**/*.neat')
# 将数据转成 document 对象，每个文件会作为一个 document
documents = loader.load()

# 初始化加载器
text_splitter = CharacterTextSplitter(chunk_size=100, chunk_overlap=0)
# 切割加载的 document
split_docs = text_splitter.split_documents(documents)

# 初始化 embeddings 对象
embedding = HuggingFaceInstructEmbeddings()
# 将 document 通过 embeddings 对象计算 embedding 向量信息并存入 Chroma 向量数据库，用于后续匹配查询
vector_store_path = r"./vector_store"

docsearch = Chroma(persist_directory=vector_store_path, embedding_function=embeddings)

https://zhuanlan.zhihu.com/p/622017658?utm_id=0&wd=&eqid=d858064200101da3000000036464a33d
https://www.jianshu.com/p/f26c7191944d
https://zhuanlan.zhihu.com/p/621769313 
https://python.langchain.com/en/latest/getting_started/getting_started.html
https://python.langchain.com/en/latest/ 
https://github.com/hwchase17/langchain 
https://python.langchain.com/en/latest/getting_started/tutorials.html
https://zhuanlan.zhihu.com/p/622017658?utm_id=0 

pip install InstructorEmbedding
pip install sentence_transformers

from InstructorEmbedding import INSTRUCTOR
model = INSTRUCTOR('hkunlp/instructor-large')
sentence = "3D ActionSLAM: wearable person tracking in multi-floor environments"
instruction = "Represent the Science title:"
embeddings = model.encode([[instruction,sentence]])
print(embeddings)

from sklearn.metrics.pairwise import cosine_similarity
from InstructorEmbedding import INSTRUCTOR
model = INSTRUCTOR('hkunlp/instructor-large')

sentences_a = [
    [
        'Represent the Science sentence: ',
        'Parton energy loss in QCD matter'
    ],
    [
        'Represent the Financial statement: ',
        'The Federal Reserve on Wednesday raised its benchmark interest rate.'
    ]
]
sentences_b = [
    [
        'Represent the Science sentence: ',
        'The Chiral Phase Transition in Dissipative Dynamics'
    ],
    [
        'Represent the Financial statement: ',
        'The funds rose less than 0.5 per cent on Friday'
    ]
]
embeddings_a = model.encode(sentences_a)
embeddings_b = model.encode(sentences_b)
similarities = cosine_similarity(embeddings_a,embeddings_b)
print(similarities)

import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from InstructorEmbedding import INSTRUCTOR
model = INSTRUCTOR('hkunlp/instructor-large')

query = [
    [
        'Represent the Wikipedia question for retrieving supporting documents: ',
        'where is the food stored in a yam plant'
    ]
]
corpus = [
    [
        'Represent the Wikipedia document for retrieval: ',
        'Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that the term "mixed economies" more precisely describes most contemporary economies, due to their containing both private-owned and state-owned enterprises. In capitalism, prices determine the demand-supply scale. For example, higher demand for certain goods and services lead to higher prices and lower demand for certain goods lead to lower prices.'
    ],
    [
        'Represent the Wikipedia document for retrieval: ',
        "The disparate impact theory is especially controversial under the Fair Housing Act because the Act regulates many activities relating to housing, insurance, and mortgage loans—and some scholars have argued that the theory's use under the Fair Housing Act, combined with extensions of the Community Reinvestment Act, contributed to rise of sub-prime lending and the crash of the U.S. housing market and ensuing global economic recession"
    ],
    [
        'Represent the Wikipedia document for retrieval: ',
        'Disparate impact in United States labor law refers to practices in employment, housing, and other areas that adversely affect one group of people of a protected characteristic more than another, even though rules applied by employers or landlords are formally neutral. Although the protected classes vary by statute, most federal civil rights laws protect based on race, color, religion, national origin, and sex as protected traits, and some laws include disability status and other traits as well.'
    ]
]
query_embeddings = model.encode(query)
corpus_embeddings = model.encode(corpus)
similarities = cosine_similarity(query_embeddings, corpus_embeddings)
print("similarities: ", similarities)
retrieved_doc_id = np.argmax(similarities)
print("retrieved_doc_id: ", retrieved_doc_id)
print("corpus[retrieved_doc_id]: ", corpus[retrieved_doc_id])

from InstructorEmbedding import INSTRUCTOR
model = INSTRUCTOR('hkunlp/instructor-large')

import sklearn.cluster
sentences = [['Represent the Medicine sentence for clustering: ','Dynamical Scalar Degree of Freedom in Horava-Lifshitz Gravity'],
             ['Represent the Medicine sentence for clustering: ','Comparison of Atmospheric Neutrino Flux Calculations at Low Energies'],
             ['Represent the Medicine sentence for clustering: ','Fermion Bags in the Massive Gross-Neveu Model'],
             ['Represent the Medicine sentence for clustering: ',"QCD corrections to Associated t-tbar-H production at the Tevatron"],
             ['Represent the Medicine sentence for clustering: ','A New Analysis of the R Measurements: Resonance Parameters of the Higher,  Vector States of Charmonium']]
embeddings = model.encode(sentences)
clustering_model = sklearn.cluster.MiniBatchKMeans(n_clusters=2)
clustering_model.fit(embeddings)
cluster_assignment = clustering_model.labels_
print(cluster_assignment)

https://huggingface.co/hkunlp/instructor-large

# 持久化数据
docsearch = Pinecone.from_texts([t.page_content for t in split_docs], embeddings, index_name=index_name)
 
# 加载数据
docsearch = Pinecone.from_existing_index(index_name, embeddings)

https://app.pinecone.io/