main

AWS official examples, with some good api call examples. https://github.com/aws-samples/rag-workshop-amazon-bedrock-knowledge-bases/tree/main

01_create_ingest_documents_test_kb_multi_ds.ipynb

Just create the KB, and get its ID to supply to configuration as kb_id

query = "Provide a summary of consolidated statements of cash flows of Octank Financial for the fiscal years ended December 31, 2019?"

foundation_model = "anthropic.claude-3-haiku-20240307-v1:0"

response = bedrock_agent_runtime_client.retrieve_and_generate(
    input={
        "text": query
    },
    retrieveAndGenerateConfiguration={
        "type": "KNOWLEDGE_BASE",
        "knowledgeBaseConfiguration": {
            'knowledgeBaseId': kb_id,
            "modelArn": "arn:aws:bedrock:{}::foundation-model/{}".format(region, foundation_model),
            "retrievalConfiguration": {
                "vectorSearchConfiguration": {
                    "numberOfResults":5
                } 
            }
        }
    }
)

print(response['output']['text'],end='\n'*2)

03_customized_rag_retreive_api_hybrid_search_langchain.ipynb

LangChain integration

# from langchain.llms.bedrock import Bedrock
import langchain
from langchain_aws import ChatBedrock
from langchain.retrievers.bedrock import AmazonKnowledgeBasesRetriever

llm = ChatBedrock(model_id=modelId, 
                  client=bedrock_client)


query = "What was the total operating lease liabilities and total sublease income of the Octank as of December 31, 2022?"
retriever = AmazonKnowledgeBasesRetriever(
        knowledge_base_id=kb_id,
        retrieval_config={"vectorSearchConfiguration": 
                          {"numberOfResults": 4,
                           'overrideSearchType': "SEMANTIC", # optional
                           }
                          },
        # endpoint_url=endpoint_url,
        # region_name=region,
        # credentials_profile_name="<profile_name>",
    )
docs = retriever.get_relevant_documents(
        query=query
    )
pp.pprint(docs)

LangChain template

from langchain.prompts import PromptTemplate

PROMPT_TEMPLATE = """
Human: You are a financial advisor AI system, and provides answers to questions by using fact based and statistical information when possible. 
Use the following pieces of information to provide a concise answer to the question enclosed in <question> tags. 
If you don't know the answer, just say that you don't know, don't try to make up an answer.
<context>
{context}
</context>

<question>
{question}
</question>

The response should be specific and use statistics or numbers when possible.

Assistant:"""
claude_prompt = PromptTemplate(template=PROMPT_TEMPLATE, 
                               input_variables=["context","question"])

04_customized_rag_retreive_api_langchain_evaluation_ragas.ipynb

provide questions and ground_truths together

import pandas as pd

questions = [
        "What was the primary reason for the increase in net cash provided by operating activities for Octank Financial in 2021?",
        "In which year did Octank Financial have the highest net cash used in investing activities, and what was the primary reason for this?",
        "What was the primary source of cash inflows from financing activities for Octank Financial in 2021?",
        "Calculate the year-over-year percentage change in cash and cash equivalents for Octank Financial from 2020 to 2021.",
        "Based on the information provided, what can you infer about Octank Financial's overall financial health and growth prospects?"
]
ground_truths = [
    "The increase in net cash provided by operating activities was primarily due to an increase in net income and favorable changes in operating assets and liabilities.",
    "Octank Financial had the highest net cash used in investing activities in 2021, at $360 million, compared to $290 million in 2020 and $240 million in 2019. The primary reason for this was an increase in purchases of property, plant, and equipment and marketable securities.",
    "The primary source of cash inflows from financing activities for Octank Financial in 2021 was an increase in proceeds from the issuance of common stock and long-term debt.",
    "To calculate the year-over-year percentage change in cash and cash equivalents from 2020 to 2021: \
    2020 cash and cash equivalents: $350 million \
    2021 cash and cash equivalents: $480 million \
    Percentage change = (2021 value - 2020 value) / 2020 value * 100 \
    = ($480 million - $350 million) / $350 million * 100 \
    = 37.14% increase",
    "Based on the information provided, Octank Financial appears to be in a healthy financial position and has good growth prospects. The company has consistently increased its net cash provided by operating activities, indicating strong profitability and efficient management of working capital. Additionally, Octank Financial has been investing in long-term assets, such as property, plant, and equipment, and marketable securities, which suggests plans for future growth and expansion. The company has also been able to finance its growth through the issuance of common stock and long-term debt, indicating confidence from investors and lenders. Overall, Octank Financial's steady increase in cash and cash equivalents over the past three years provides a strong foundation for future growth and investment opportunities."
]

answers = []
contexts = []

for query in questions:
  answers.append(qa_chain.invoke(query)["result"])
  contexts.append([docs.page_content for docs in retriever.invoke(query)])

# Create dataset df
dataset_df = pd.DataFrame(list(zip(questions, ground_truths, answers)),
                 columns=['questions', 'ground_truths', 'answers'])
# dataset_df

and evaluation

# evaluation using ragas

from tqdm.notebook import tqdm
import pandas as pd

# ragas libraries
from ragas import SingleTurnSample 
from ragas.llms import LangchainLLMWrapper
from ragas.embeddings import LangchainEmbeddingsWrapper

# ragas metrics
from ragas.metrics import (LLMContextPrecisionWithReference,
                            LLMContextRecall)
from ragas.metrics._factual_correctness import FactualCorrectness

metrics = { 'context_precision': LLMContextPrecisionWithReference,
             'context_recall': LLMContextRecall, 
             'factual_correctness': FactualCorrectness
            }

# Model Configuration
llm_for_evaluation = LangchainLLMWrapper(llm_for_evaluation)
bedrock_embeddings = LangchainEmbeddingsWrapper(bedrock_embeddings)

# Create an empty DataFrame
df_retrieve = pd.DataFrame(columns=list(metrics.keys()))

score_list = []
# Outer progress bar for samples
with tqdm(total=len(questions), desc="Evaluating samples") as pbar_samples:
    for idx, (question, ground_truth, context, answer) in enumerate(zip(questions, ground_truths, contexts, answers)):
        score_dict = {}
        # Inner progress bar for metrics
        with tqdm(total=len(metrics), desc=f"Sample {idx+1} metrics", leave=False) as pbar_metrics:
            for k, v in metrics.items():
                scorer = v(llm=llm_for_evaluation)
                sample = SingleTurnSample(
                            user_input=question,
                            response=answer,
                            reference=ground_truth,
                            retrieved_contexts=context, 
                            )
                score_result = await scorer.single_turn_ascore(sample)
                score_dict[k] = score_result
                pbar_metrics.update(1)
        score_list.append(score_dict)
        pbar_samples.update(1)

# convert score_list to df
df_metrics = pd.DataFrame(score_list)

# merge metric df with dataset df
results = pd.merge(dataset_df, df_metrics, left_index=True, right_index=True)