Od zera Semantic Search ElasticSearch jako baza wektorowa dokumentów i co dalej

Cezary Walenciuk

Od zera Semantic Search
ElasticSearch
jako baza wektorowa dokumentów i co dalej V2

@walenciukC

Moja próba tokenizacji w C#

                    

























































































                        namespace SemanticSearch.Models
                        {
                            using System;
                            using System.Collections.Generic;
                            using System.Linq;
                        
                            /// <summary>
                            /// Word piece tokenizer based on bert-base-uncased model in transformers.
                            /// See tools/tokenizer.py for examples in python.
                            /// </summary>
                            public class BertTokenizer
                            {
                                private readonly List<string> vocabulary;
                        
                                public BertTokenizer(List<string> vocabulary)
                                {
                                    this.vocabulary = vocabulary;
                                }
                        
                                /// <summary>
                                /// Tokenize a set of strings.
                                /// </summary>
                                /// <param name="texts">List of strings.</param>
                                /// <returns>List of tokens.</returns>
                                public List<(string Token, int VocabularyIndex)> Tokenize(IEnumerable<string> texts)
                                {
                                    // [CLS] Words of sentence [SEP] Words of next sentence [SEP]
                                    IEnumerable<string> tokens = new[]
                                        {
                                            DefaultTokens.Classification
                                        };
                        
                                    foreach (var text in texts)
                                    {
                                        tokens = tokens.Concat(this.TokenizeSentence(text));
                                        tokens = tokens.Concat(new[] { DefaultTokens.Separation });
                                    }
                        
                                    return tokens
                                        .SelectMany(this.TokenizeSubwords)
                                        .ToList();
                                }
                        
                                /**
                                 * Some words in the vocabulary are too big and will be broken up in to subwords
                                 * Example "Embeddings"
                                 * [‘em’, ‘##bed’, ‘##ding’, ‘##s’]
                                 * https://mccormickml.com/2019/05/14/BERT-word-embeddings-tutorial/
                                 * https://developpaper.com/bert-visual-learning-of-the-strongest-nlp-model/
                                 * https://medium.com/@_init_/why-bert-has-3-embedding-layers-and-their-implementation-details-9c261108e28a
                                 */
                                private IEnumerable<(string Token, int VocabularyIndex)>
                                    TokenizeSubwords(string word)
                                {
                                    if (this.vocabulary.Contains(word))
                                    {
                                        return new (string, int)[]
                                        {
                                            (word, this.vocabulary.IndexOf(word))
                                        };
                                    }
                        
                                    var tokens = new List<(string, int)>();
                                    var remaining = word;
                        
                                    while (!string.IsNullOrEmpty(remaining) && remaining.Length > 2)
                                    {
                                        var prefix = this.vocabulary.Where(remaining.StartsWith)
                                            .OrderByDescending(o => o.Length)
                                            .FirstOrDefault();
                        
                                        if (prefix == null)
                                        {
                                            tokens.Add((DefaultTokens.Unknown,
                                                this.vocabulary.IndexOf(DefaultTokens.Unknown)));
                        
                                            return tokens;
                                        }
                        
                                        var replaced = remaining.Replace(prefix, "##");
                                        if (replaced.Length == remaining.Length)
                                        {
                                            break;
                                        }
                        
                                        remaining = replaced;
                        
                                        tokens.Add((prefix, this.vocabulary.IndexOf(prefix)));
                                    }
                        
                                    if (!string.IsNullOrWhiteSpace(word) && !tokens.Any())
                                    {
                                        tokens.Add((DefaultTokens.Unknown,
                                            this.vocabulary.IndexOf(DefaultTokens.Unknown)));
                                    }
                        
                                    return tokens;
                                }
                        
                                private IEnumerable<string> TokenizeSentence(string text)
                                {
                                    // remove spaces and split the , . : ; etc..
                                    return text.Split(
                                        new[] { " ", "   ", "\r\n" },
                                        StringSplitOptions.None)
                                        .SelectMany(o =>
                                                o.SplitAndKeep(".,;:\\/?!#$%()=+-*\"'–_`<>&^@{}[]|~'"
                                                .ToArray()))
                                        .Select(o => o.ToLower());
                                }
                        
                                public class DefaultTokens
                                {
                                    public const string Padding = "";
                                    public const string Unknown = "[UNK]";
                                    public const string Classification = "[CLS]";
                                    public const string Separation = "[SEP]";
                                    public const string Mask = "[MASK]";
                                }
                            }
                        }

Tworzenie indeksu my_index_demo_vector z polem dense_vector

                    
                        PUT my_index_demo_vector
                        {
                         "mappings": {
                           "properties": {
                             "title": {
                             "type": "text"
                             },
                             "embedding": {
                             "type": "dense_vector",
                             "dims": 16
                             }
                           }
                         }
                        }

Dodanie dokumentu do my_index_demo_vector

                    
                        POST  my_index_demo_vector/_doc/1
                        {
                         "title": "Polisa ubezpieczeniowa potężnego Franka",
                         "embedding": [0.123, 0.456, 0.789, 0.234, 0.567, 0.890,
                          0.345, 0.678, 0.901, 0.123, 0.456, 0.789, 0.234, 0.567, 0.890, 0.345]
                        }

Zapytanie hybrydowe do ElasticSearch

                    
                        POST  my_index_demo_vector/_search
                        {
                           "query": {
                                    "script_score": {
                                        "query": {
                                                "match": {
                                                        "title": "polisa ubezpieczeniowa"
                                                }
                                        },
                                        "script": {
                                            "source": "cosineSimilarity(params.queryVector, 'embedding') + 1.0",
                                            "params": {
                                            "queryVector": [0.123, 0.456, 0.789, 0.234, 0.567, 0.890, 0.345,
                                             0.678, 0.901, 0.123, 0.456, 0.789, 0.234, 0.567, 0.890, 0.345]
                                            }
                                        }    
                                    }
                           }
                        }

Wynik zapytania

                    










                        {
                            "took": 1,
                            "timed_out": false,
                            "_shards": {
                              "total": 1,
                              "successful": 1,
                              "skipped": 0,
                              "failed": 0
                            },
                            "hits": {
                              "total": {
                                "value": 1,
                                "relation": "eq"
                              },
                              "max_score": 2,
                              "hits": [
                                {
                                  "_index": "my_index_demo_vector",
                                  "_id": "1",
                                  "_score": 2,
                                  "_source": {
                                    "title": "Polisa ubezpieczeniowa potężnego Franka",
                                    "embedding": [
                                      0.123,
                                      0.456,
                                      0.789,....
                                    ]
                                  }
                                }
                              ]
                            }
                          }

Sprawdzenie czy mogę odpytać ElasticSearch w Pythonie

                    
                        from elasticsearch import Elasticsearch

                        es = Elasticsearch(
                            "http://localhost:9201"
                        )
                        es.ping()

Odczytanie pliku CSV

                    
                        import pandas as pd

                        df = pd.read_csv("products_catalog.csv").loc[:499]
                        df.head()

                        df.fillna("None", inplace=True)

Dodanie do tabelki kolumny "DescriptionVector"

                    
                        from sentence_transformers import SentenceTransformer

                        model = SentenceTransformer('all-mpnet-base-v2')

                        df["DescriptionVector"] = df["Description"].apply(lambda x: model.encode(x))

Tworzenie indeksu w ElasticSearch

                    
                        from indexMapping import indexMapping

                        es.indices.create(index="all_products_demo", mappings=indexMapping)

Kod struktury indeksu

                    























                        indexMapping = {
                            "properties":{
                                "ProductID":{
                                    "type":"long"
                                },
                                "ProductName":{
                                    "type":"text"
                                },
                                "ProductBrand":{
                                    "type":"text"
                                },
                                "Gender":{
                                    "type":"text"
                                },
                                "Price (INR)":{
                                    "type":"long"
                                },
                                "NumImages":{
                                    "type":"long"
                                },
                                "Description":{
                                    "type":"text"
                                },
                                "PrimaryColor":{
                                    "type":"text"
                                },
                                "DescriptionVector":{
                                    "type":"dense_vector",
                                    "dims": 768,
                                    "index":True,
                                    "similarity": "l2_norm"
                                }
                        
                            }
                        }

Zapisanie rekordów do ElasticSearch

                    
                        record_list = df.to_dict("records")

                        for record in record_list:
                            try:
                                es.index(index="all_products_demo", document=record, 
                                    id=record["ProductID"])
                            except Exception as e:
                                print(e)

                        es.count(index="all_products")

Tworzenie zapytania do ElasticSearch

                    
                        input_keyword = "Blue Shoes"

                        vector_of_input_keyword = model.encode(input_keyword)
                        
                        query = {
                            "field" : "DescriptionVector",
                            "query_vector" : vector_of_input_keyword,
                            "k" : 2,
                            "num_candidates" : 500, 
                        }

                        #res = es.knn_search(index="all_products_demo", knn=query , source=["ProductName","Description"])
                        #ElasticsearchWarning: The kNN search API has been replaced by the `knn` option in the search API.

                        res2 = es.search(index="all_products_demo",knn=query, source=["ProductName","Description"])
                        res2["hits"]["hits"]

Odpowiedź z ElasticSearch

                                                                                                  True             32
                        Name: count, dtype: int64
                        ObjectApiResponse({'acknowledged': True, 'shards_acknowledged': True, 'index': 'all_products_demo'})
                        ObjectApiResponse({'count': 468, '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0}})
                        [{'_index': 'all_products_demo',
                          '_id': '10018013',
                          '_score': 0.61429423,
                          '_source': {'ProductName': 'Puma Men Blue Sneakers',
                           'Description': 'A pair of round-toe blue sneakers, has regular styling, lace-up detailTextile upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer'}},
                         {'_index': 'all_products_demo',
                          '_id': '10018075',
                          '_score': 0.61429423,
                          '_source': {'ProductName': 'Puma Men Blue Sneakers',
                           'Description': 'A pair of round-toe blue sneakers, has regular styling, lace-up detailTextile upperCushioned footbedTextured and patterned outsoleWarranty: 3 monthsWarranty provided by brand/manufacturer'}}]

Kod wyszukiwarki

                    








                        







































                        import streamlit as st
                        from elasticsearch import Elasticsearch
                        from sentence_transformers import SentenceTransformer
                        
                        indexName = "all_products_demo"
                        
                        try:
                            es = Elasticsearch(
                            "http://localhost:9201"
                            )
                        except ConnectionError as e:
                            print("Connection Error:", e)
                            
                        if es.ping():
                            print("Succesfully connected to ElasticSearch!!")
                        else:
                            print("Oops!! Can not connect to Elasticsearch!")
                        
                        
                        
                        
                        def search(input_keyword):
                            model = SentenceTransformer('all-mpnet-base-v2')
                            vector_of_input_keyword = model.encode(input_keyword)
                        
                            query = {
                                "field": "DescriptionVector",
                                "query_vector": vector_of_input_keyword,
                                "k": 10,
                                "num_candidates": 500
                            }
                            res = es.knn_search(index="all_products_demo"
                                                , knn=query 
                                                , source=["ProductName","Description"]
                                                )
                            results = res["hits"]["hits"]
                        
                            return results
                        
                        def main():
                            st.title("Search Fashion Products")
                        
                            # Input: User enters search query
                            search_query = st.text_input("Enter your search query")
                        
                            # Button: User triggers the search
                            if st.button("Search"):
                                if search_query:
                                    # Perform the search and get results
                                    results = search(search_query)
                        
                                    # Display search results
                                    st.subheader("Search Results")
                                    for result in results:
                                        with st.container():
                                            if '_source' in result:
                                                try:
                                                    st.header(f"{result['_source']['ProductName']}")
                                                except Exception as e:
                                                    print(e)
                                                
                                                try:
                                                    st.write(f"Description: {result['_source']['Description']}")
                                                except Exception as e:
                                                    print(e)
                                                st.divider()
                        
                                            
                        if __name__ == "__main__":
                            main()

OpenAI

                    
                        from openai import OpenAI
                        client = OpenAI(api_key="sk-I3DkQxDvx000jEei3Bcfvsdsdsfdfcdsfasd3IoDOC")
                        
                        def get_embedding(text, model="text-embedding-ada-002"):
                           text = text.replace("\n", " ")
                           return client.embeddings.create(input = [text], model=model).data[0].embedding

Zapytanie do ChatGTP

                    
                        input_keyword = "brown Shoes for men under 1500"
                        vector_of_input_keyword = get_embedding(input_keyword)

                        my_prompt = f"""I have data in elastic search of all clothing products with their description, 
                            color, price and the gender they belongs to.
                            genders are {gender_list}
                            colors are {color_list}
                            price can be anything from 0 to 100k
                            based on user's search query. give me json output as follows
                            {{
                            "color": "it should be what users want. give Not-Mentioned if user did not explicitly 
                            mentioned the color in query. If the color mentioned by user is not present in above color list, give Not-Found",
                            "gender": "gender should be from above list only. if not specified give Not-Mentioned."
                            "max_price":
                            "min_price":
                            }}

                            users query : {input_keyword}
                            """

Wysłanie zapytania do ChatGTP

                    
                        response = client.chat.completions.create(
                            model="gpt-3.5-turbo-1106",
                            response_format={ "type": "json_object" },
                            messages=[
                              {"role": "system",
                                "content": 
                                "You are a helpful assistant designed to output only in JSON format.No other text or explaination."},
                              {"role": "user", "content": my_prompt}
                            ]
                        )

Wyciąganie odpowiedzi z OpenAI

                    
                        response.choices[0].message.content
                       
                        filter_map = json.loads(response.choices[0].message.content)
                        # {'color': 'Brown', 'gender': 'Men', 'max_price': 1500, 'min_price': 0}

Tworzenie zapytania hybrydowego do ElasticSearch

                    

















                        q1 = {
                            "knn": {
                                "field": "NameDescriptionVector",
                                "query_vector": vector_of_input_keyword,
                                "k": 10,
                                "num_candidates": 10000
                            },
                            "_source": ["ProductName","Description","PrimaryColor","Price (INR)","ProductBrand","Gender"]
                        }
                        
                        
                        filter_query = {
                            "bool": {
                                "must": [
                                    {
                                        "match": {
                                            "PrimaryColor": {
                                                "query": filter_map["color"],
                                                "fuzzy_transpositions": "false",
                                                "fuzziness": 0
                                            }
                                        }
                                    },
                                    {
                                        "range": {
                                            "Price (INR)": {
                                                "gte": filter_map["min_price"],
                                                "lte": filter_map["max_price"]
                                            }
                                        }
                                    }
                                ]
                            }
                        }
                        
                        res = es.knn_search(index="my_products_demo2",
                                            body=q1,
                                            request_timeout=5000,
                                            filter=filter_query)
                        
                        res["hits"]["hits"]