使用 Vertex AI 自动建立公共网站索引以实现高效语义搜索

# Authenticate to your GCP project

gcloud auth login

# Enable following services

# Vertex AI

gcloud services enable aiplatform.googleapis.com --async

# Cloud Run

gcloud services enable run.googleapis.com

# Cloud Logging

gcloud logging enable

# Resource Manager

gcloud services enable cloudresourcemanager.googleapis.com

# Artifact Registry

gcloud services enable artifactregistry.googleapis.com

# Storage

gcloud services enable storage.googleapis.com

git clone https://github.com/IvanERufino/public-website-vector-search

gcloud builds submit --tag gcr.io/public-website-index-administrator/data_pipeline

# Update the cloud run parameters for memory and instances if needed

gcloud run deploy datapipeline --image 

gcr.io/public-website-index-administrator/data_pipeline --allow-unauthenticated --region=asia-east1 --memory=2Gi --min-instances=1

curl -XPOST -H 'Content-Type: application/json' https://datapipeline.web.app/api/v1/public_url -d '{"index_name": "testing", "url": "https://cloud.google.com/bigquery/sitemap.xml", "prefix_name": "bigquery"}'

from langchain.vectorstores.matching_engine import MatchingEngine

from langchain.agents import Tool

from langchain.embeddings import VertexAIEmbeddings

from vertexai.preview.language_models import TextGenerationModel

from google.cloud import aiplatform

PROJECT_ID = '{YOUR-PROJECT-ID}'

REGION = '{YOUR-REGION}'

INDEX='{YOUR-INDEX-ID}'

ENDPOINT='{YOUR-INDEX-ENDPOINT-ID}'

DOCS_BUCKET='{YOUR-GCS-BUCKET}'

embeddings = VertexAIEmbeddings()

vector_store = MatchingEngine.from_components(

    index_id=INDEX,

    region=REGION,

    embedding=embeddings,

    project_id=PROJECT_ID,

    endpoint_id=ENDPOINT,

    gcs_bucket_name=DOCS_BUCKET

)

def matching_engine_search(question):

    relevant_documentation=vector_store.similarity_search(question, k=8)

    context = "\\n".join([doc.page_content for doc in relevant_documentation])[:10000]

    return str(context)

TEXT_GENERATION_MODEL='text-bison@001'

question = "What is Apigee"

matching_engine_response=matching_engine_search(question)

prompt=f"""

Follow exactly those 3 steps:

1. Read the context below and aggregate this data

Context : {matching_engine_response}

2. Answer the question using only this context

3. Return the answer

User Question: {question}

If you don't have any context and are unsure of the answer, reply that you don't know about this topic.

"""

model = TextGenerationModel.from_pretrained(TEXT_GENERATION_MODEL)

response = model.predict(

        prompt,

        temperature=0.2,

        top_k=40,

        top_p=.8,

        max_output_tokens=1024,

)

print(f"Question: \\n{question}")

print(f"Response: \\n{response.text}")