TextSearch API

*(a::Dict{Ti,Tv}, b::Dict{Ti,Tv}) where {Ti,Tv<:Real}
*(a::Dict{K, V}, b::F) where K where {V<:Real} where {F<:Real}

Computes the element-wise product of a and b

+(a::Dict{Ti,Tv}, b::Dict{Ti,Tv}) where {Ti,Tv<:Real}
+(a::Dict, b::Pair)

Computes the sum of a and b

-(a::Dict{Ti,Tv}, b::Dict{Ti,Tv}) where {Ti,Tv<:Real}

Substracts of b of a

/(a::Dict{K, V}, b::F) where K where {V<:Real} where {F<:Real}

Computes the element-wise division of a and b

Base.sum(col::AbstractVector{<:Dict})

Computes the sum of the given list of vectors

zero(::Type{Dict{Ti,Tv}}) where {Ti,Tv}

Creates an empty Dict vector

evaluate(::Angle, a::Dict, b::Dict)::Float64

Computes the angle between two Dict sparse vectors

evaluate(::Cosine, a::Dict, b::Dict)::Float64

Computes the cosine distance between two Dict sparse vectors

evaluate(::NormAngle, a::Dict, b::Dict)::Float64

Computes the angle between two Dict sparse vectors

It supposes that all bags are normalized (see normalize! function)

evaluate(::NormCosine, a::Dict, b::Dict)::Float64

Computes the cosine distance between two Dict sparse vectors

It supposes that bags are normalized (see normalize! function)

dot(a::Dict, b::Dict)

Computes the dot product for two Dict vectors

norm(a::Dict)

Computes a normalized Dict vector

normalize!(bow::Dict)

Inplace normalization of bow

search(acceptpostinglist::Function, idx::BM25InvertedFile, ctx::InvertedFileContext, qtext::AbstractString, res::AbstractKnn search(idx::BM25InvertedFile, ctx::InvertedFileContext, qtext::AbstractString, res::AbstractKnn

Find candidates for solving query Q using idx. It calls callback on each candidate (docID, dist)

sparsevec(vec::Dict{Ti,Tv}, m=0) where {Ti<:Integer,Tv<:Number}

Creates a sparse vector from a Dict-based sparse vector

add!(a::Dict{Ti,Tv}, b::Dict{Ti,Tv}) where {Ti,Tv<:Real}
add!(a::Dict{Ti,Tv}, b::AbstractSparseArray) where {Ti,Tv<:Real}
add!(a::Dict{Ti,Tv}, b::Pair{Ti,Tv}) where {Ti,Tv<:Real}

Updates a to the sum of a+b

bagofwords!(bow::BOW, voc::Vocabulary, tokenlist::TokenizedText)
bagofwords!(buff::TextSearchBuffer, voc::Vocabulary, text)
bagofwords(voc::Vocabulary, messages)

Creates a bag of words from the given text (a string or a list of strings). If bow is given then updates the bag with the text. When config is given, the text is parsed according to it.

bagofwords(voc::Vocabulary, messages)
bagofwords!(buff, voc::Vocabulary, messages)

Computes a bag of words from messages

bagofwords_corpus(voc::Vocabulary, corpus::AbstractVector; minbatch=0)

Computes a list of bag of words from a corpus

centroid(cluster::AbstractVector{<:Dict})

Computes a centroid of the given list of Dict vectors

collocations(q, buff::TextSearchBuffer, tt::AbstractTokenTransformation, mark_token_type)

Computes a kind of collocations of the given text

dvec(x::AbstractSparseVector)

Converts an sparse vector into a dict-based sparse vector

filter_tokens!(voc::Vocabulary, text::TokenizedText)

Removes tokens from text array

filter_tokens!(voc::Vocabulary, text::TokenizedText)

Removes tokens from a given tokenized text based using the valid vocabulary

filter_tokens(pred::Function, voc::Vocabulary)

Returns a copy of reduced vocabulary based on evaluating pred function for each entry in voc

flush_collocations!(buff::TextSearchBuffer, tt::AbstractTokenTransformation, mark_token_type)

Pushes a collocation inside the buffer to the token list; it discards empty strings.

flush_nword!(buff::TextSearchBuffer, tt::AbstractTokenTransformation, mark_token_type)

Pushes the nword inside the buffer to the token list; it discards empty strings.

flush_qgram!(buff::TextSearchBuffer, tt::AbstractTokenTransformation, mark_token_type)

Pushes the qgram inside the buffer to the token list; it discards empty strings.

flush_skipgram!(buff::TextSearchBuffer, tt::AbstractTokenTransformation, mark_token_type)

Pushes the skipgram inside the buffer to the token list; it discards empty strings.

flush_unigram!(buff::TextSearchBuffer, tt::AbstractTokenTransformation)

Pushes the word inside the buffer to the token list; it discards empty strings.

merge_voc(voc1::Vocabulary, voc2::Vocabulary[, ...])
merge_voc(pred::Function, voc1::Vocabulary, voc2::Vocabulary[, ...])

Merges two or more vocabularies into a new one. A predicate function can be used to filter token entries.

Note: All vocabularies should had been created with a compatible TextConfig to be able to work on them.

normalize_text(config::TextConfig, text::AbstractString, output::Vector{Char}; limits::Bool=true)

Normalizes a given text using the specified transformations of config

nwords(q::Integer, buff::TextSearchBuffer, tt::AbstractTokenTransformation, mark_token_type)

qgrams(q::Integer, buff::TextSearchBuffer, tt::AbstractTokenTransformation, mark_token_type)

Computes character q-grams for the given input

skipgrams(q::Skipgram, buff::TextSearchBuffer, tt::AbstractTokenTransformation, mark_token_type)

Tokenizes using skipgrams

sparse(cols::AbstractVector{<:Dict}, m=0; minweight=1e-9) 
sparse_coo(cols::AbstractVector{<:Dict}, minweight=1e-9)

Creates a sparse matrix from an array of Dict sparse vectors.

tokenize(textconfig::TextConfig, text)
tokenize(copy_::Function, textconfig::TextConfig, text)

tokenize(textconfig::TextConfig, text, buff)
tokenize(copy_::Function, textconfig::TextConfig, text, buff)

Tokenizes text using the given configuration. The tokenize makes heavy usage of buffers, and when these buffers are shared it is mandatory to create a copy of the result (buff.tokens).

Change the default copy function to make an additional filtering of the tokens. You can also pass the identity function to avoid copying.

tokenize_and_append!(voc::Vocabulary, corpus; minbatch=0)

Parse each document in the given corpus and appends each token to the vocabulary.

tokenize_corpus(textconfig::TextConfig, arr; minbatch=0, verbose=true)
tokenize_corpus(copy_::Function, textconfig::TextConfig, arr; minbatch=0, verbose=true)

Tokenize a list of texts. The copy_ function is passed to tokenize as first argument.

transform_collocation(::AbstractTokenTransformation, tok)

Hook applied in the tokenization stage to change the input token tok if needed. Return nothing to ignore the tok occurence (e.g., stop words).

transform_nword(::AbstractTokenTransformation, tok)

Hook applied in the tokenization stage to change the input token tok if needed. For instance, it can be used to apply stemming or any other kind of normalization. Return nothing to ignore the tok occurence (e.g., stop words).

transform_qgram(::AbstractTokenTransformation, tok)

Hook applied in the tokenization stage to change the input token tok if needed. For instance, it can be used to apply stemming or any other kind of normalization. Return nothing to ignore the tok occurence (e.g., stop words).

transform_skipgram(::AbstractTokenTransformation, tok)

Hook applied in the tokenization stage to change the input token tok if needed. For instance, it can be used to apply stemming or any other kind of normalization. Return nothing to ignore the tok occurence (e.g., stop words).

transform_unigram(::AbstractTokenTransformation, tok)

Hook applied in the tokenization stage to change the input token tok if needed. For instance, it can be used to apply stemming or any other kind of normalization. Return nothing to ignore the tok occurence (e.g., stop words).

unigrams(buff::TextSearchBuffer, tt::AbstractTokenTransformation)

Performs the word tokenization

update_voc!(voc::Vocabulary, another::Vocabulary)
update_voc!(pred::Function, voc::Vocabulary, another::Vocabulary)

Update voc vocabulary using another vocabulary. Optionally a predicate can be given to filter vocabularies.

Note 1: corpuslen remains unchanged (the structure is immutable and a new Vocabulary should be created to update this field). Note 2: Both voc and another vocabularies should had been created with a compatible TextConfig to be able to work on them.

vectorize!(buff::TextSearchBuffer, model::VectorModel{G_,L_}, bow::BOW; normalize=true, minweight=1e-9) where {G_,L_}

Computes a weighted vector using the given bag of words and the specified weighting scheme.

Vocabulary(textconfig, corpus; minbatch=0)

Computes a vocabulary from a corpus using the TextConfig textconfig.

struct BM25InvertedFile <: AbstractInvertedFile

Parameters

BM25InvertedFile(voc::Vocabulary, db=nothing)

Fits the BM25 score from the given vocabulary it also creates the associated inverted file structure.

BinaryGlobalWeighting()

The weight is 1 for known tokens, 0 for out of vocabulary tokens

BinaryLocalWeighting()

The weight is 1 for known tokens, 0 for out of vocabulary tokens

EntropyWeighting(; smooth=0.0, lowerweight=0.0, weights=:balance)

Entropy weighting uses the empirical entropy of the vocabulary along classes to produce a notion of importance for each token

FreqWeighting()

Frequency weighting

GlobalWeighting

Abstract type for global weighting

IdfWeighting()

Inverse document frequency weighting

LocalWeighting

Abstract type for local weighting

Skipgram(qsize, skip)

A skipgram is a kind of tokenization where qsize words having skip separation are used as a single token.

TextConfig(;
    del_diac::Bool=true,
    del_dup::Bool=false,
    del_punc::Bool=false,
    group_num::Bool=true,
    group_url::Bool=true,
    group_usr::Bool=false,
    group_emo::Bool=false,
    lc::Bool=true,
    collocations::Int8=0,
    qlist::Vector=Int8[],
    nlist::Vector=Int8[],
    slist::Vector{Skipgram}=Skipgram[],
    mark_token_type::Bool = true
    tt=IdentityTokenTransformation()
)

Defines a preprocessing and tokenization pipeline

• del_diac: indicates if diacritic symbols should be removed
• del_dup: indicates if duplicate contiguous symbols must be replaced for a single symbol
• del_punc: indicates if punctuaction symbols must be removed
• group_num: indicates if numbers should be grouped _num
• group_url: indicates if urls should be grouped as _url
• group_usr: indicates if users (@usr) should be grouped as _usr
• group_emo: indicates if emojis should be grouped as _emo
• lc: indicates if the text should be normalized to lower case
• collocations: window to expand collocations as tokens, please take into account that:
  • 0 => disables collocations
  • 1 => will compute words (ignored in favor of use typical unigrams)
  • 2 => will compute bigrams (don't use this, but not disabled)
  • 3 <= typical values
• qlist: a list of character q-grams to use
• nlist: a list of words n-grams to use
• slist: a list of skip-grams tokenizers to use
• mark_token_type: each token is marked with its type (qgram, skipgram, nword) when is true.
• tt: An AbstractTokenTransformation struct

Note: If qlist, nlist, and slists are all empty arrays, then it defaults to nlist=[1]

Model

An abstract type that represents a weighting model

TfWeighting()

Term frequency weighting

TpWeighting()

Term probability weighting

VectorModel(ent::EntropyWeighting, lw::LocalWeighting, corpus::BOW, labels;
    mindocs::Integer=1,
    smooth::Float64=0.0,
    weights=:balance
    comb::CombineWeighting=NormalizedEntropy(),
)

Creates a vector model using the input corpus.

Vocabulary(textconfig::TextConfig, trainsize::Int, numtokens::Int)

Creates a Vocabulary struct