Alibaba CloudにてMinIOというS3準拠オブジェクトストレージを立ち上げて爆速でS3-RAGサービスを作る

wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20240716234641.0.0_amd64.deb -O minio.deb
dpkg -i minio.deb

tee /etc/systemd/system/minio.service <<EOF

[Unit]
Description=MinIO
Documentation=https://min.io/docs/minio/linux/index.html
Wants=network-online.target
After=network-online.target
AssertFileIsExecutable=/usr/local/bin/minio

[Service]
WorkingDirectory=/usr/local
User=minio-user
Group=minio-user
ProtectProc=invisible

EnvironmentFile=-/etc/default/minio
ExecStartPre=/bin/bash -c "if [ -z \"${MINIO_VOLUMES}\" ]; then echo \"Variable MINIO_VOLUMES not set in /etc/default/minio\"; exit 1; fi"
ExecStart=/usr/local/bin/minio server $MINIO_OPTS $MINIO_VOLUMES

Restart=always
LimitNOFILE=65536
TasksMax=infinity
TimeoutStopSec=infinity
SendSIGKILL=no

[Install]
WantedBy=multi-user.target
EOF

groupadd -r minio-user
useradd -M -r -g minio-user minio-user
mkdir -p /mnt/disk{1..4}
chown minio-user:minio-user /mnt/disk{1..4}

tee /etc/default/minio <<EOF

MINIO_VOLUMES="http://10.2.0.29:9000/mnt/disk{1..4}/minio http://10.2.0.28:9000/mnt/disk{1..4}/minio http://10.2.0.27:9000/mnt/disk{1..4}/minio"

MINIO_OPTS="--console-address :9001"

MINIO_ROOT_USER=minioadmin

MINIO_ROOT_PASSWORD=minio-secret-key-CHANGE-ME

EOF

root@iZ6wea00iajgu2rabxroi9Z:~# lsblk 
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINTS 
loop0 7:0 0 61.9M 1 loop /snap/core20/1405 
loop1 7:1 0 79.9M 1 loop /snap/lxd/22923 
loop3 7:3 0 38.8M 1 loop /snap/snapd/21759 
loop4 7:4 0 63.9M 1 loop /snap/core20/2318 
loop5 7:5 0 87M 1 loop /snap/lxd/29351 
vda 252:0 0 100G 0 disk 
├─vda1 252:1 0 1M 0 part 
├─vda2 252:2 0 200M 0 part /boot/efi 
└─vda3 252:3 0 99.8G 0 part / 
root@iZ6wea00iajgu2rabxroi9Z:~# 
root@iZ6wea00iajgu2rabxroi9Z:~# mount | grep /mnt 
root@iZ6wea00iajgu2rabxroi9Z:~#

mkdir -p /mnt/disk{1..4}
dd if=/dev/zero of=/mnt/disk1/disk.img bs=1M count=1024 
dd if=/dev/zero of=/mnt/disk2/disk.img bs=1M count=1024 
dd if=/dev/zero of=/mnt/disk3/disk.img bs=1M count=1024 
dd if=/dev/zero of=/mnt/disk4/disk.img bs=1M count=1024 
mkfs.ext4 /mnt/disk1/disk.img sudo mkfs.ext4 /mnt/disk2/disk.img 
mkfs.ext4 /mnt/disk3/disk.img sudo mkfs.ext4 /mnt/disk4/disk.img 
mount -o loop /mnt/disk1/disk.img /mnt/disk1 
mount -o loop /mnt/disk2/disk.img /mnt/disk2 
mount -o loop /mnt/disk3/disk.img /mnt/disk3 
mount -o loop /mnt/disk4/disk.img /mnt/disk4

sudo systemctl daemon-reload 
systemctl restart minio.service

import boto3
import os        
from datasets import load_dataset


s3_resource = boto3.resource(
    's3',
    endpoint_url='http://47.xx.xxx.244:9000',
    aws_access_key_id='kFSxFDn1QFN1GxvrMK0o',
    aws_secret_access_key='xxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
)


s3_resource.create_bucket(Bucket="rag-service")

# Hugging Faceのデータセットをロード
dataset = load_dataset("AhmedSSabir/Japanese-wiki-dump-sentence-dataset")


# データセットを一時的にローカル保存し、MinIOへアップロード
for split in dataset.keys():
    split_dataset = dataset[split]
    local_file_path = f"/tmp/{split}.jsonl"    
    split_dataset.to_json(local_file_path, orient='records', lines=True)    
    s3_resource.Bucket(bucket_name).upload_file(local_file_path, f"{split}.jsonl")    
    os.remove(local_file_path)

import openai
import chromadb
import os
import sys
from langchain_community.document_loaders import S3DirectoryLoader
from langchain.chains import ConversationalRetrievalChain, RetrievalQA
from langchain_community.chat_models import ChatOpenAI
from langchain_community.llms import OpenAI
from langchain.text_splitter import CharacterTextSplitter
from langchain_community.vectorstores import Chroma
from langchain.indexes.vectorstore import VectorStoreIndexWrapper
from langchain_openai import OpenAIEmbeddings

# OpenAI APIキーの設定
os.environ["OPENAI_API_KEY"] = ""

query = sys.argv[1] if len(sys.argv) > 1 else None

# S3バケットからドキュメントをロード
s3_loader = S3DirectoryLoader(
                "s3",
                endpoint_url="http://47.xx.xxx.244:9000",
                aws_access_key_id="kFSxFDn1QFN1GxvrMK0o",
                aws_secret_access_key="xxxxxxxxxxxxxxxxxxxxxxxxxxxxx")
docs = s3_loader.load()
splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
split_texts = splitter.split_documents(docs)

# ドキュメントのベクトル化と検索
embeddings = OpenAIEmbeddings()
vector_store = Chroma.from_documents(split_texts, embeddings)

# QAオブジェクトの作成
qa_system = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=vector_store.as_retriever())

# 会話型検索チェーンの作成
conv_chain = ConversationalRetrievalChain.from_llm(
    llm=ChatOpenAI(model="gpt-3.5-turbo"),
    retriever=vector_store.as_retriever(search_kwargs={"k": 1}),
)

chat_history = []
while True:
    if not query:
        query = input("プロンプト: ")
    if query in ['quit', 'q', 'exit']:
        sys.exit()
    result = conv_chain({"question": query, "chat_history": chat_history})
    print(result['answer'])
    chat_history.append((query, result['answer']))
    query = None