[Langchain] Prompt 튜닝

출처 : https://bcho.tistory.com/1414

Langchain을 이용한 LLM 애플리케이션 개발 #8 - 프롬프트 예제 선택기

 

Prompt 튜닝

• Prompt 통해 정확도를 높이기 위한 방법 >>  질문과 답변에 예제를 추가하는 방식
• 기본원리는 여러개의 예제를 저장해 놓고, 질문에 따라서 가장 유사한 예제를 선택하는 방식 (Example Selector 제공)

 

Length 기반

• 전체 Prompt의 길이 기반

#################################################################
# Class : ExampleSelector - ExampleSelectorLength
#################################################################
class ExampleSelectorLength(ExampleSelector):
    className  = 'Length'
    paramName  = 'max_length'

    def getSelector(self):
        return LengthBasedExampleSelector(
            examples       = self.objExamples, 
            example_prompt = self.objExamplePrompt, 
            max_length     = self.paramValue
        )

    def getExamples(self):
        return [
        {"food":"Kimchi"   ,"category":"Korean food" },
        {"food":"Chocolate","category":"dessert"     },
        {"food":"Pasta"    ,"category":"Italian food"},
        {"food":"Americano","category":"Coffee"      }
    ]

 

 

 

N-gram Overlap

• 연속된 N개의 아이템(단어/문자)으로 이루어진 순서 있는 집합을 질문과 비교하여 중첩되는 수를 기준으로 순위 정함
  - 질문과 더 많이 중첩되는 예제를 선정

#################################################################
# Class : ExampleSelector - ExampleSelectorLength
#################################################################
class ExampleSelectorNGram(ExampleSelector):
    className  = 'NGram'
    paramName  = 'threshold'

    def getSelector(self):        
        return NGramOverlapExampleSelector(
            examples       = self.objExamples, 
            example_prompt = self.objExamplePrompt, 
            threshold      = self.paramValue
        )
    
    def getExamples(self):
        return [
        {"food":"Kimchi"   ,"category":"Korean food" },
        {"food":"Chocolate","category":"dessert"     },
        {"food":"Pasta"    ,"category":"Italian food"},
        {"food":"Americano","category":"Coffee"      }
    ]

 

 

 

유사도 기반

• 예제와 질문을 임베딩(벡터값으로 변경)하여 가장 비슷한 예제를 선정하는 방식
• OpenAI의 Embedding API 이용
• 벡터DB로 chroma 이용

#################################################################
# Class : ExampleSelector - EampleSelectorSemantic
#################################################################
class ExampleSelectorSemantic(ExampleSelector):
    className  = 'Semantic'
    paramName  = 'k-value'

    def getSelector(self):        
        return SemanticSimilarityExampleSelector.from_examples(
            self.objExamples, 
            OpenAIEmbeddings(), 
            Chroma, 
            k=1,
        )
    
    def getExamples(self):
        return [
            {"input":"Happy."  ,"category":"emotion"},
            {"input":"BBQ"     ,"category":"food"   },
            {"input":"Golf"    ,"category":"Sports" },
            {"input":"Student" ,"category":"Person" }
        ]

 

 

통계요약 알고리즘 기반 (MMR)

• 유사도 기반의 다양성 문제를 해결하기 위한 방식 (즉, 여러 질문인 경우에 대한 대응)

#################################################################
# Class : ExampleSelector - EampleSelectorMMR
#################################################################
class ExampleSelectorSemantic(ExampleSelector):
    className  = 'MMR'
    paramName  = 'k-value'

    def getSelector(self):        
        return MaxMarginalRelevanceExampleSelector.from_examples(
            self.objExamples, 
            OpenAIEmbeddings(), 
            Chroma, 
            k=self.paramValue,
        )
    
    def getExamples(self):
        return [
            {"input"  :"Please summarize the weather news.\n",
            "summary":"Today's weather: Sunny skies, mild temperatures, and a gentle breeze. Enjoy the pleasant conditions throughout the day!"},
            {"input"  :"Please summarize the economy news.\n",
            "summary":"Global stocks rise on positive economic data; inflation concerns persist. Tech sector outperforms; central banks closely monitor."},
            {"input"  :"Please summarize retail news.\n",
            "summary":"Major retailer announces record-breaking sales during holiday shopping season"},
            {"input"  :"What is stock market trend?\n",
            "summary":"Investor optimism grows amid easing global trade tensions"},
            {"input"  :"Typhoon related news.\n",
            "summary":"IAirports and schools close ahead of approaching typhoon threat"},   
        ]

 

 

 

나머지 부분 -1  (import library)

 
from langchain_openai import OpenAIEmbeddings
from langchain.prompts import FewShotPromptTemplate, PromptTemplate

from langchain.prompts.example_selector import (
    MaxMarginalRelevanceExampleSelector, 
    SemanticSimilarityExampleSelector, 
    LengthBasedExampleSelector, 
    NGramOverlapExampleSelector
)
from langchain_community.vectorstores import Chroma
 
import os

 

 

나머지 부분 -2 (base class 정의)

#################################################################
# Class : ExampleSelector
#################################################################
class ExampleSelector:
    className  = 'None'
    paramName  = 'None'
    paramValue = None


    def __init__(self):
        pass
    
    def getPrompt(self, varParam, varTemplate, varInTemplate, varPrefix, varSuffix, varInput):
        self.paramValue         = varParam
        self.varTemplate        = varTemplate
        self.varInTemplate      = varInTemplate
        self.varPrefix          = varPrefix
        self.varSuffix          = varSuffix
        self.varInput           = varInput

        self.objExamples        = self.getExamples()
        self.objExamplePrompt   = self.getExamplePrompt()
        self.objExampleSelector = self.getSelector()
        objPrompt               = self.getPromptFromTemplate()
        return objPrompt
        

    def printPrompt(self, objPrompt):
        print( '\n', '-'*50)
        print(f'- class     : {self.className}')
        print(f'- parameter : {self.paramName} - {self.paramValue}')
        print( '- Prompt    :')
        print( '-'*50)
        print(objPrompt)
        print( '')

    def getExamplePrompt(self):
        return PromptTemplate(
            template       = self.varTemplate,
            input_variables= self.varInTemplate
        )

    def getPromptFromTemplate(self):
        return FewShotPromptTemplate(
            example_selector = self.objExampleSelector,
            example_prompt   = self.objExamplePrompt,
            prefix           = self.varPrefix,
            suffix           = self.varSuffix,
            input_variables  = self.varInput
        )

 

 

나머지 부분 -3 (실제 main)

#################################################################
# Main function 
#################################################################
# 


if __name__ == '__main__':

    # # Length base
    # classExampleSelector = ExampleSelectorLength()
    # objFinalPrompt       = classExampleSelector.getPrompt(40, "Food:{food} Category:{category}", ["food","category"], "What is the category of this food?", "Food: {food}", ["food"])
    # classExampleSelector.printPrompt(objFinalPrompt.format(food = 'Korean BBQ'))

    # # N-gram base
    # classExampleSelector = ExampleSelectorNGram()
    # objFinalPrompt       = classExampleSelector.getPrompt(0.0,"Food:{food} Category:{category}", ["food","category"], "What is the category of this food?", "Food: {food}", ["food"])
    # classExampleSelector.printPrompt(objFinalPrompt.format(food = 'Korean BBQ'))

    # # Semanctic base 
    # classExampleSelector = ExampleSelectorSemantic()
    # objFinalPrompt       = classExampleSelector.getPrompt(1  ,"Input:{input} Category:{category}", ["input","category"], "What is the category of the input? .", "input: {input}", ["input"])
    # classExampleSelector.printPrompt(objFinalPrompt.format(input = 'Sushi'))

    # MMR
    classExampleSelector = ExampleSelectorSemantic()
    objFinalPrompt       = classExampleSelector.getPrompt(2  ,"Input:{input} Summary:{summary}", ["input","summary"], "", "input: {input}\nSummary:", ["input"])
    classExampleSelector.printPrompt(objFinalPrompt.format(input="I want to know the economy trends and weather this week."))