LightRAG
1.0.0
該存儲庫託管Lightrag的代碼。該代碼的結構基於納米 - 圖形。
textract 。 圖1：Lightrag索引流程圖圖2：Lightrag檢索和查詢流程圖
cd LightRAG
pip install -e .pip install lightrag-hkuexamples中找到。export OPENAI_API_KEY="sk-...".curl https://raw.githubusercontent.com/gusye1234/nano-graphrag/main/tests/mock_data.txt > ./book.txt使用以下Python片段（在腳本中）初始化Lightrag並執行查詢：
import os
from lightrag import LightRAG , QueryParam
from lightrag . llm import gpt_4o_mini_complete , gpt_4o_complete
#########
# Uncomment the below two lines if running in a jupyter notebook to handle the async nature of rag.insert()
# import nest_asyncio
# nest_asyncio.apply()
#########
WORKING_DIR = "./dickens"
if not os . path . exists ( WORKING_DIR ):
os . mkdir ( WORKING_DIR )
rag = LightRAG (
working_dir = WORKING_DIR ,
llm_model_func = gpt_4o_mini_complete # Use gpt_4o_mini_complete LLM model
# llm_model_func=gpt_4o_complete # Optionally, use a stronger model
)
with open ( "./book.txt" ) as f :
rag . insert ( f . read ())
# Perform naive search
print ( rag . query ( "What are the top themes in this story?" , param = QueryParam ( mode = "naive" )))
# Perform local search
print ( rag . query ( "What are the top themes in this story?" , param = QueryParam ( mode = "local" )))
# Perform global search
print ( rag . query ( "What are the top themes in this story?" , param = QueryParam ( mode = "global" )))
# Perform hybrid search
print ( rag . query ( "What are the top themes in this story?" , param = QueryParam ( mode = "hybrid" ))) async def llm_model_func (
prompt , system_prompt = None , history_messages = [], ** kwargs
) -> str :
return await openai_complete_if_cache (
"solar-mini" ,
prompt ,
system_prompt = system_prompt ,
history_messages = history_messages ,
api_key = os . getenv ( "UPSTAGE_API_KEY" ),
base_url = "https://api.upstage.ai/v1/solar" ,
** kwargs
)
async def embedding_func ( texts : list [ str ]) -> np . ndarray :
return await openai_embedding (
texts ,
model = "solar-embedding-1-large-query" ,
api_key = os . getenv ( "UPSTAGE_API_KEY" ),
base_url = "https://api.upstage.ai/v1/solar"
)
rag = LightRAG (
working_dir = WORKING_DIR ,
llm_model_func = llm_model_func ,
embedding_func = EmbeddingFunc (
embedding_dim = 4096 ,
max_token_size = 8192 ,
func = embedding_func
)
) from lightrag . llm import hf_model_complete , hf_embedding
from transformers import AutoModel , AutoTokenizer
from lightrag . utils import EmbeddingFunc
# Initialize LightRAG with Hugging Face model
rag = LightRAG (
working_dir = WORKING_DIR ,
llm_model_func = hf_model_complete , # Use Hugging Face model for text generation
llm_model_name = 'meta-llama/Llama-3.1-8B-Instruct' , # Model name from Hugging Face
# Use Hugging Face embedding function
embedding_func = EmbeddingFunc (
embedding_dim = 384 ,
max_token_size = 5000 ,
func = lambda texts : hf_embedding (
texts ,
tokenizer = AutoTokenizer . from_pretrained ( "sentence-transformers/all-MiniLM-L6-v2" ),
embed_model = AutoModel . from_pretrained ( "sentence-transformers/all-MiniLM-L6-v2" )
)
),
)如果您想使用Ollama型號，則需要拉動計劃使用和嵌入模型的模型，例如nomic-embed-text 。
然後，您只需要按以下方式設置Lightrag：
from lightrag . llm import ollama_model_complete , ollama_embedding
from lightrag . utils import EmbeddingFunc
# Initialize LightRAG with Ollama model
rag = LightRAG (
working_dir = WORKING_DIR ,
llm_model_func = ollama_model_complete , # Use Ollama model for text generation
llm_model_name = 'your_model_name' , # Your model name
# Use Ollama embedding function
embedding_func = EmbeddingFunc (
embedding_dim = 768 ,
max_token_size = 8192 ,
func = lambda texts : ollama_embedding (
texts ,
embed_model = "nomic-embed-text"
)
),
) export NEO4J_URI = "neo4j://localhost:7687"
export NEO4J_USERNAME = "neo4j"
export NEO4J_PASSWORD = "password"
When you launch the project be sure to override the default KG : NetworkS
by specifying kg = "Neo4JStorage" .
# Note: Default settings use NetworkX
#Initialize LightRAG with Neo4J implementation.
WORKING_DIR = "./local_neo4jWorkDir"
rag = LightRAG (
working_dir = WORKING_DIR ,
llm_model_func = gpt_4o_mini_complete , # Use gpt_4o_mini_complete LLM model
kg = "Neo4JStorage" , #<-----------override KG default
log_level = "DEBUG" #<-----------override log_level default
)有關工作示例，請參見test_neo4j.py。
為了使Lightrag工作環境至少應為32K令牌。默認情況下，Ollama模型的上下文大小為8K。您可以使用兩種方法之一來實現這一目標：
num_ctx參數。ollama pull qwen2ollama show --modelfile qwen2 > ModelfilePARAMETER num_ctx 32768ollama create -f Modelfile qwen2mnum_ctx 。 tiy可以使用llm_model_kwargs param進行配置Ollama：
rag = LightRAG (
working_dir = WORKING_DIR ,
llm_model_func = ollama_model_complete , # Use Ollama model for text generation
llm_model_name = 'your_model_name' , # Your model name
llm_model_kwargs = { "options" : { "num_ctx" : 32768 }},
# Use Ollama embedding function
embedding_func = EmbeddingFunc (
embedding_dim = 768 ,
max_token_size = 8192 ,
func = lambda texts : ollama_embedding (
texts ,
embed_model = "nomic-embed-text"
)
),
)有功能齊全的示例examples/lightrag_ollama_demo.py使用gemma2:2b模型，僅在並行運行4個請求，並將上下文大小設置為32K。
為了在低RAM GPU上運行此實驗，您應該選擇小型模型並調整上下文窗口（增加上下文增加了內存消耗）。例如，在使用gemma2:2b時，在重新利用的挖掘GPU上運行此Ollama示例，將上下文大小設置為26K。它能夠在book.txt上找到197個實體和19個關係。
class QueryParam :
mode : Literal [ "local" , "global" , "hybrid" , "naive" ] = "global"
only_need_context : bool = False
response_type : str = "Multiple Paragraphs"
# Number of top-k items to retrieve; corresponds to entities in "local" mode and relationships in "global" mode.
top_k : int = 60
# Number of tokens for the original chunks.
max_token_for_text_unit : int = 4000
# Number of tokens for the relationship descriptions
max_token_for_global_context : int = 4000
# Number of tokens for the entity descriptions
max_token_for_local_context : int = 4000 # Batch Insert: Insert multiple texts at once
rag . insert ([ "TEXT1" , "TEXT2" ,...]) # Incremental Insert: Insert new documents into an existing LightRAG instance
rag = LightRAG (
working_dir = WORKING_DIR ,
llm_model_func = llm_model_func ,
embedding_func = EmbeddingFunc (
embedding_dim = embedding_dimension ,
max_token_size = 8192 ,
func = embedding_func ,
),
)
with open ( "./newText.txt" ) as f :
rag . insert ( f . read ()) rag = LightRAG (
working_dir = WORKING_DIR ,
llm_model_func = llm_model_func ,
embedding_func = EmbeddingFunc (
embedding_dim = embedding_dimension ,
max_token_size = 8192 ,
func = embedding_func ,
),
)
custom_kg = {
"entities" : [
{
"entity_name" : "CompanyA" ,
"entity_type" : "Organization" ,
"description" : "A major technology company" ,
"source_id" : "Source1"
},
{
"entity_name" : "ProductX" ,
"entity_type" : "Product" ,
"description" : "A popular product developed by CompanyA" ,
"source_id" : "Source1"
}
],
"relationships" : [
{
"src_id" : "CompanyA" ,
"tgt_id" : "ProductX" ,
"description" : "CompanyA develops ProductX" ,
"keywords" : "develop, produce" ,
"weight" : 1.0 ,
"source_id" : "Source1"
}
]
}
rag . insert_custom_kg ( custom_kg ) # Delete Entity: Deleting entities by their names
rag = LightRAG (
working_dir = WORKING_DIR ,
llm_model_func = llm_model_func ,
embedding_func = EmbeddingFunc (
embedding_dim = embedding_dimension ,
max_token_size = 8192 ,
func = embedding_func ,
),
)
rag . delete_by_entity ( "Project Gutenberg" )textract支持讀取文件類型，例如TXT，DOCX，PPTX，CSV和PDF。
import textract
file_path = 'TEXT.pdf'
text_content = textract . process ( file_path )
rag . insert ( text_content . decode ( 'utf-8' ))examples/graph_visual_with_html.py中找到 import networkx as nx
from pyvis . network import Network
# Load the GraphML file
G = nx . read_graphml ( './dickens/graph_chunk_entity_relation.graphml' )
# Create a Pyvis network
net = Network ( notebook = True )
# Convert NetworkX graph to Pyvis network
net . from_nx ( G )
# Save and display the network
net . show ( 'knowledge_graph.html' )examples/graph_visual_with_neo4j.py中找到 import os
import json
from lightrag . utils import xml_to_json
from neo4j import GraphDatabase
# Constants
WORKING_DIR = "./dickens"
BATCH_SIZE_NODES = 500
BATCH_SIZE_EDGES = 100
# Neo4j connection credentials
NEO4J_URI = "bolt://localhost:7687"
NEO4J_USERNAME = "neo4j"
NEO4J_PASSWORD = "your_password"
def convert_xml_to_json ( xml_path , output_path ):
"""Converts XML file to JSON and saves the output."""
if not os . path . exists ( xml_path ):
print ( f"Error: File not found - { xml_path } " )
return None
json_data = xml_to_json ( xml_path )
if json_data :
with open ( output_path , 'w' , encoding = 'utf-8' ) as f :
json . dump ( json_data , f , ensure_ascii = False , indent = 2 )
print ( f"JSON file created: { output_path } " )
return json_data
else :
print ( "Failed to create JSON data" )
return None
def process_in_batches ( tx , query , data , batch_size ):
"""Process data in batches and execute the given query."""
for i in range ( 0 , len ( data ), batch_size ):
batch = data [ i : i + batch_size ]
tx . run ( query , { "nodes" : batch } if "nodes" in query else { "edges" : batch })
def main ():
# Paths
xml_file = os . path . join ( WORKING_DIR , 'graph_chunk_entity_relation.graphml' )
json_file = os . path . join ( WORKING_DIR , 'graph_data.json' )
# Convert XML to JSON
json_data = convert_xml_to_json ( xml_file , json_file )
if json_data is None :
return
# Load nodes and edges
nodes = json_data . get ( 'nodes' , [])
edges = json_data . get ( 'edges' , [])
# Neo4j queries
create_nodes_query = """
UNWIND $nodes AS node
MERGE (e:Entity {id: node.id})
SET e.entity_type = node.entity_type,
e.description = node.description,
e.source_id = node.source_id,
e.displayName = node.id
REMOVE e:Entity
WITH e, node
CALL apoc.create.addLabels(e, [node.entity_type]) YIELD node AS labeledNode
RETURN count(*)
"""
create_edges_query = """
UNWIND $edges AS edge
MATCH (source {id: edge.source})
MATCH (target {id: edge.target})
WITH source, target, edge,
CASE
WHEN edge.keywords CONTAINS 'lead' THEN 'lead'
WHEN edge.keywords CONTAINS 'participate' THEN 'participate'
WHEN edge.keywords CONTAINS 'uses' THEN 'uses'
WHEN edge.keywords CONTAINS 'located' THEN 'located'
WHEN edge.keywords CONTAINS 'occurs' THEN 'occurs'
ELSE REPLACE(SPLIT(edge.keywords, ',')[0], ' " ', '')
END AS relType
CALL apoc.create.relationship(source, relType, {
weight: edge.weight,
description: edge.description,
keywords: edge.keywords,
source_id: edge.source_id
}, target) YIELD rel
RETURN count(*)
"""
set_displayname_and_labels_query = """
MATCH (n)
SET n.displayName = n.id
WITH n
CALL apoc.create.setLabels(n, [n.entity_type]) YIELD node
RETURN count(*)
"""
# Create a Neo4j driver
driver = GraphDatabase . driver ( NEO4J_URI , auth = ( NEO4J_USERNAME , NEO4J_PASSWORD ))
try :
# Execute queries in batches
with driver . session () as session :
# Insert nodes in batches
session . execute_write ( process_in_batches , create_nodes_query , nodes , BATCH_SIZE_NODES )
# Insert edges in batches
session . execute_write ( process_in_batches , create_edges_query , edges , BATCH_SIZE_EDGES )
# Set displayName and labels
session . run ( set_displayname_and_labels_query )
except Exception as e :
print ( f"Error occurred: { e } " )
finally :
driver . close ()
if __name__ == "__main__" :
main ()| 範圍 | 類型 | 解釋 | 預設 |
|---|---|---|---|
| working_dir | str | 將存儲緩存的目錄 | lightrag_cache+timestamp |
| kv_storage | str | 文檔和文本塊的存儲類型。支持類型： JsonKVStorage ， OracleKVStorage | JsonKVStorage |
| vector_storage | str | 嵌入向量的存儲類型。支持類型： NanoVectorDBStorage ， OracleVectorDBStorage | NanoVectorDBStorage |
| graph_storage | str | 圖形邊緣和節點的存儲類型。支持類型： NetworkXStorage ， Neo4JStorage ， OracleGraphStorage | NetworkXStorage |
| log_level | 應用程序運行時的日誌級別 | logging.DEBUG | |
| chunk_token_size | int | 分割文件時每塊的最大令牌尺寸 | 1200 |
| chunk_overlap_token_size | int | 分裂文檔時兩個塊之間的重疊令牌大小 | 100 |
| tiktoken_model_name | str | 用於計算令牌編號的Tiktoken編碼器的模型名稱 | gpt-4o-mini |
| ENTITY_EXTRACT_MAX_GLEANING | int | 實體提取過程中的循環數量，附加歷史記錄消息 | 1 |
| entity_summary_to_max_tokens | int | 每個實體摘要的最大令牌大小 | 500 |
| node_embedding_algorithm | str | 節點嵌入算法（當前未使用） | node2vec |
| node2vec_params | dict | 節點嵌入的參數 | {"dimensions": 1536,"num_walks": 10,"walk_length": 40,"window_size": 2,"iterations": 3,"random_seed": 3,} |
| embedding_func | EmbeddingFunc | 從文本生成嵌入向量的功能 | openai_embedding |
| embedding_batch_num | int | 嵌入過程的最大批量大小（每批發送多個文本） | 32 |
| embedding_func_max_async | int | 同步異步嵌入過程的最大數量 | 16 |
| llm_model_func | callable | LLM生成的功能 | gpt_4o_mini_complete |
| llm_model_name | str | llm型號的生成名稱 | meta-llama/Llama-3.2-1B-Instruct |
| llm_model_max_token_size | int | LLM生成的最大令牌大小（影響實體關係摘要） | 32768 |
| llm_model_max_async | int | 最大並發異步LLM過程的數量 | 16 |
| llm_model_kwargs | dict | LLM生成的其他參數 | |
| vector_db_storage_cls_kwargs | dict | 向量數據庫的其他參數（當前未使用） | |
| enable_llm_cache | bool | 如果為TRUE ，則存儲LLM會導致緩存；重複提示返回緩存的響應 | TRUE |
| addon_params | dict | 其他參數，例如{"example_number": 1, "language": "Simplified Chinese"} ：設置示例限制和輸出語言 | example_number: all examples, language: English |
| convert_response_to_json_func | callable | 未使用 | convert_response_to_json |
Lightrag還提供了一個基於FastAPI的服務器實現，以供RAG操作恢復API訪問。這使您可以運行Lightrag作為服務，並通過HTTP請求與它進行交互。
pip install fastapi uvicorn pydantic export RAG_DIR= " your_index_directory " # Optional: Defaults to "index_default"
export OPENAI_BASE_URL= " Your OpenAI API base URL " # Optional: Defaults to "https://api.openai.com/v1"
export OPENAI_API_KEY= " Your OpenAI API key " # Required
export LLM_MODEL= " Your LLM model " # Optional: Defaults to "gpt-4o-mini"
export EMBEDDING_MODEL= " Your embedding model " # Optional: Defaults to "text-embedding-3-large"python examples/lightrag_api_openai_compatible_demo.py服務器將從http://0.0.0.0:8020開始。
API服務器提供以下端點：
/query{
"query" : " Your question here " ,
"mode" : " hybrid " , // Can be "naive", "local", "global", or "hybrid"
"only_need_context" : true // Optional: Defaults to false, if true, only the referenced context will be returned, otherwise the llm answer will be returned
}curl -X POST " http://127.0.0.1:8020/query "
-H " Content-Type: application/json "
-d ' {"query": "What are the main themes?", "mode": "hybrid"} ' /insert{
"text" : " Your text content here "
}curl -X POST " http://127.0.0.1:8020/insert "
-H " Content-Type: application/json "
-d ' {"text": "Content to be inserted into RAG"} ' /insert_file{
"file_path" : " path/to/your/file.txt "
}curl -X POST " http://127.0.0.1:8020/insert_file "
-H " Content-Type: application/json "
-d ' {"file_path": "./book.txt"} ' /healthcurl -X GET " http://127.0.0.1:8020/health "可以使用環境變量配置API服務器：
RAG_DIR ：存儲抹布索引的目錄（默認值：“ index_default”）API包括全面的錯誤處理：
Lightrag中使用的數據集可以從Tommychien/Ultradomain下載。
Lightrag使用以下提示來生成高級查詢，並在example/generate_query.py中使用相應的代碼。
Given the following description of a dataset :
{ description }
Please identify 5 potential users who would engage with this dataset . For each user , list 5 tasks they would perform with this dataset . Then , for each ( user , task ) combination , generate 5 questions that require a high - level understanding of the entire dataset .
Output the results in the following structure :
- User 1 : [ user description ]
- Task 1 : [ task description ]
- Question 1 :
- Question 2 :
- Question 3 :
- Question 4 :
- Question 5 :
- Task 2 : [ task description ]
...
- Task 5 : [ task description ]
- User 2 : [ user description ]
...
- User 5 : [ user description ]
...為了評估兩個抹布系統在高級查詢上的性能，Lightrag使用以下提示，其中提供了example/batch_eval.py中的特定代碼。
- - - Role - - -
You are an expert tasked with evaluating two answers to the same question based on three criteria : ** Comprehensiveness ** , ** Diversity ** , and ** Empowerment ** .
- - - Goal - - -
You will evaluate two answers to the same question based on three criteria : ** Comprehensiveness ** , ** Diversity ** , and ** Empowerment ** .
- ** Comprehensiveness ** : How much detail does the answer provide to cover all aspects and details of the question ?
- ** Diversity ** : How varied and rich is the answer in providing different perspectives and insights on the question ?
- ** Empowerment ** : How well does the answer help the reader understand and make informed judgments about the topic ?
For each criterion , choose the better answer ( either Answer 1 or Answer 2 ) and explain why . Then , select an overall winner based on these three categories .
Here is the question :
{ query }
Here are the two answers :
** Answer 1 : **
{ answer1 }
** Answer 2 : **
{ answer2 }
Evaluate both answers using the three criteria listed above and provide detailed explanations for each criterion .
Output your evaluation in the following JSON format :
{{
"Comprehensiveness" : {{
"Winner" : "[Answer 1 or Answer 2]" ,
"Explanation" : "[Provide explanation here]"
}},
"Empowerment" : {{
"Winner" : "[Answer 1 or Answer 2]" ,
"Explanation" : "[Provide explanation here]"
}},
"Overall Winner" : {{
"Winner" : "[Answer 1 or Answer 2]" ,
"Explanation" : "[Summarize why this answer is the overall winner based on the three criteria]"
}}
}}| 農業 | CS | 合法的 | 混合 | |||||
|---|---|---|---|---|---|---|---|---|
| naiverag | Lightrag | naiverag | Lightrag | naiverag | Lightrag | naiverag | Lightrag | |
| 全面性 | 32.4％ | 67.6％ | 38.4％ | 61.6％ | 16.4％ | 83.6％ | 38.8％ | 61.2％ |
| 多樣性 | 23.6％ | 76.4％ | 38.0％ | 62.0％ | 13.6％ | 86.4％ | 32.4％ | 67.6％ |
| 授權 | 32.4％ | 67.6％ | 38.8％ | 61.2％ | 16.4％ | 83.6％ | 42.8％ | 57.2％ |
| 全面的 | 32.4％ | 67.6％ | 38.8％ | 61.2％ | 15.2％ | 84.8％ | 40.0％ | 60.0％ |
| RQ-rag | Lightrag | RQ-rag | Lightrag | RQ-rag | Lightrag | RQ-rag | Lightrag | |
| 全面性 | 31.6％ | 68.4％ | 38.8％ | 61.2％ | 15.2％ | 84.8％ | 39.2％ | 60.8％ |
| 多樣性 | 29.2％ | 70.8％ | 39.2％ | 60.8％ | 11.6％ | 88.4％ | 30.8％ | 69.2％ |
| 授權 | 31.6％ | 68.4％ | 36.4％ | 63.6％ | 15.2％ | 84.8％ | 42.4％ | 57.6％ |
| 全面的 | 32.4％ | 67.6％ | 38.0％ | 62.0％ | 14.4％ | 85.6％ | 40.0％ | 60.0％ |
| 海德 | Lightrag | 海德 | Lightrag | 海德 | Lightrag | 海德 | Lightrag | |
| 全面性 | 26.0％ | 74.0％ | 41.6％ | 58.4％ | 26.8％ | 73.2％ | 40.4％ | 59.6％ |
| 多樣性 | 24.0％ | 76.0％ | 38.8％ | 61.2％ | 20.0％ | 80.0％ | 32.4％ | 67.6％ |
| 授權 | 25.2％ | 74.8％ | 40.8％ | 59.2％ | 26.0％ | 74.0％ | 46.0％ | 54.0％ |
| 全面的 | 24.8％ | 75.2％ | 41.6％ | 58.4％ | 26.4％ | 73.6％ | 42.4％ | 57.6％ |
| GraphRag | Lightrag | GraphRag | Lightrag | GraphRag | Lightrag | GraphRag | Lightrag | |
| 全面性 | 45.6％ | 54.4％ | 48.4％ | 51.6％ | 48.4％ | 51.6％ | 50.4％ | 49.6％ |
| 多樣性 | 22.8％ | 77.2％ | 40.8％ | 59.2％ | 26.4％ | 73.6％ | 36.0％ | 64.0％ |
| 授權 | 41.2％ | 58.8％ | 45.2％ | 54.8％ | 43.6％ | 56.4％ | 50.8％ | 49.2％ |
| 全面的 | 45.2％ | 54.8％ | 48.0％ | 52.0％ | 47.2％ | 52.8％ | 50.4％ | 49.6％ |
所有代碼都可以在./reproduce目錄中找到。
首先，我們需要在數據集中提取獨特的上下文。
def extract_unique_contexts ( input_directory , output_directory ):
os . makedirs ( output_directory , exist_ok = True )
jsonl_files = glob . glob ( os . path . join ( input_directory , '*.jsonl' ))
print ( f"Found { len ( jsonl_files ) } JSONL files." )
for file_path in jsonl_files :
filename = os . path . basename ( file_path )
name , ext = os . path . splitext ( filename )
output_filename = f" { name } _unique_contexts.json"
output_path = os . path . join ( output_directory , output_filename )
unique_contexts_dict = {}
print ( f"Processing file: { filename } " )
try :
with open ( file_path , 'r' , encoding = 'utf-8' ) as infile :
for line_number , line in enumerate ( infile , start = 1 ):
line = line . strip ()
if not line :
continue
try :
json_obj = json . loads ( line )
context = json_obj . get ( 'context' )
if context and context not in unique_contexts_dict :
unique_contexts_dict [ context ] = None
except json . JSONDecodeError as e :
print ( f"JSON decoding error in file { filename } at line { line_number } : { e } " )
except FileNotFoundError :
print ( f"File not found: { filename } " )
continue
except Exception as e :
print ( f"An error occurred while processing file { filename } : { e } " )
continue
unique_contexts_list = list ( unique_contexts_dict . keys ())
print ( f"There are { len ( unique_contexts_list ) } unique `context` entries in the file { filename } ." )
try :
with open ( output_path , 'w' , encoding = 'utf-8' ) as outfile :
json . dump ( unique_contexts_list , outfile , ensure_ascii = False , indent = 4 )
print ( f"Unique `context` entries have been saved to: { output_filename } " )
except Exception as e :
print ( f"An error occurred while saving to the file { output_filename } : { e } " )
print ( "All files have been processed." )對於提取的上下文，我們將它們插入Lightrag系統中。
def insert_text ( rag , file_path ):
with open ( file_path , mode = 'r' ) as f :
unique_contexts = json . load ( f )
retries = 0
max_retries = 3
while retries < max_retries :
try :
rag . insert ( unique_contexts )
break
except Exception as e :
retries += 1
print ( f"Insertion failed, retrying ( { retries } / { max_retries } ), error: { e } " )
time . sleep ( 10 )
if retries == max_retries :
print ( "Insertion failed after exceeding the maximum number of retries" )我們從數據集中每個上下文的第一個和後半部分提取令牌，然後將它們組合為數據集說明以生成查詢。
tokenizer = GPT2Tokenizer . from_pretrained ( 'gpt2' )
def get_summary ( context , tot_tokens = 2000 ):
tokens = tokenizer . tokenize ( context )
half_tokens = tot_tokens // 2
start_tokens = tokens [ 1000 : 1000 + half_tokens ]
end_tokens = tokens [ - ( 1000 + half_tokens ): 1000 ]
summary_tokens = start_tokens + end_tokens
summary = tokenizer . convert_tokens_to_string ( summary_tokens )
return summary對於第2步中生成的查詢，我們將提取它們並查詢Lightrag。
def extract_queries ( file_path ):
with open ( file_path , 'r' ) as f :
data = f . read ()
data = data . replace ( '**' , '' )
queries = re . findall ( r'- Question d+: (.+)' , data )
return queries .
├── examples
│ ├── batch_eval . py
│ ├── generate_query . py
│ ├── graph_visual_with_html . py
│ ├── graph_visual_with_neo4j . py
│ ├── lightrag_api_openai_compatible_demo . py
│ ├── lightrag_azure_openai_demo . py
│ ├── lightrag_bedrock_demo . py
│ ├── lightrag_hf_demo . py
│ ├── lightrag_lmdeploy_demo . py
│ ├── lightrag_ollama_demo . py
│ ├── lightrag_openai_compatible_demo . py
│ ├── lightrag_openai_demo . py
│ ├── lightrag_siliconcloud_demo . py
│ └── vram_management_demo . py
├── lightrag
│ ├── kg
│ │ ├── __init__ . py
│ │ └── neo4j_impl . py
│ ├── __init__ . py
│ ├── base . py
│ ├── lightrag . py
│ ├── llm . py
│ ├── operate . py
│ ├── prompt . py
│ ├── storage . py
│ └── utils . py
├── reproduce
│ ├── Step_0 . py
│ ├── Step_1_openai_compatible . py
│ ├── Step_1 . py
│ ├── Step_2 . py
│ ├── Step_3_openai_compatible . py
│ └── Step_3 . py
├── . gitignore
├── . pre - commit - config . yaml
├── Dockerfile
├── get_all_edges_nx . py
├── LICENSE
├── README . md
├── requirements . txt
├── setup . py
├── test_neo4j . py
└── test . py 感謝我們所有的貢獻者！
@ article { guo2024lightrag ,
title = { LightRAG : Simple and Fast Retrieval - Augmented Generation },
author = { Zirui Guo and Lianghao Xia and Yanhua Yu and Tu Ao and Chao Huang },
year = { 2024 },
eprint = { 2410.05779 },
archivePrefix = { arXiv },
primaryClass = { cs . IR }
}感謝您對我們的工作的興趣！
