# This file is evaluated after the unmodified vendored simstring_pure.rb. # Keep these correctness fixes visible rather than silently editing upstream. require 'json' module SimString class DiceMeasure # Dice = 2 * common / (query_size + y_size), not their product. def minimum_common_feature_count(query_size, y_size, alpha) (0.5 * alpha * (query_size + y_size)).ceil.to_i end end class StringMatcher private # The original CPMerge misses matches when tau == 1, because the suffix # scan is empty. Collect candidates already meeting tau in the prefix too. def overlap_join(query_feature_set, tau, db, y_size) return [] if tau > query_feature_set.size postings = query_feature_set.to_h { |feature| [feature, get(db, y_size, feature)] } features = query_feature_set.sort_by { |feature| postings[feature].size } counts = Hash.new(0) features.first(features.size - tau + 1).each do |feature| postings[feature].each { |string| counts[string] += 1 } end matches = counts.select { |_, count| count >= tau }.keys matches.each { |string| counts.delete(string) } ((features.size - tau + 1)...features.size).each do |index| counts.keys.each do |string| counts[string] += 1 if postings[features[index]].include?(string) if counts[string] >= tau matches << string counts.delete(string) elsif counts[string] + features.size - index - 1 < tau counts.delete(string) end end end matches end end end module SimStringBrowser MEASURES = { 'cosine' => SimString::CosineMeasure, 'dice' => SimString::DiceMeasure, 'jaccard' => SimString::JaccardMeasure, 'overlap' => SimString::OverlapMeasure, 'exact' => SimString::ExactMeasure, }.freeze def self.search(options) query = options.fetch('query').strip raise 'Enter a search of up to 120 characters.' if query.empty? || query.length > 120 n = Integer(options.fetch('n')) raise 'Choose an n-gram length from 1 to 4.' unless (1..4).cover?(n) threshold = Float(options.fetch('threshold')) raise 'Choose a similarity from 0.10 to 1.00.' unless threshold.finite? && (0.1..1.0).cover?(threshold) measure = MEASURES.fetch(options.fetch('measure')).new values = options.fetch('values') raise 'Use a list of up to 6,000 strings.' unless values.is_a?(Array) && (1..6000).cover?(values.size) raise 'Keep each list item to 120 characters.' unless values.all? { |value| value.is_a?(String) && value.length <= 120 } ignore_case = options.fetch('ignoreCase') == true normalized = ignore_case ? query.downcase : query key = [values, n, ignore_case] rebuilt = key != @key if rebuilt extractor = SimString::NGramBuilder.new(n) labels = {} values.each do |value| value = value.strip next if value.empty? labels[ignore_case ? value.downcase : value] ||= value end raise 'Add at least one nonempty list item.' if labels.empty? database = SimString::Database.new(extractor) labels.each_key { |value| database.add(value) } @extractor, @labels, @database = extractor, labels, database @key = key end query_features = @extractor.features(normalized) # Avoid rejecting an exact boundary match when ceil/floor in the index # bounds rounds a floating-point product just above an integer. Retrieve # with a tiny allowance, then apply the requested score cutoff exactly. matches = SimString::StringMatcher.new(@database, measure).ranked_search(normalized, threshold - 1e-12) matches.select! { |match| match.score >= threshold } # Resolve score ties consistently across native Ruby and the WASM build. matches.sort_by! { |match| [-match.score, @labels.fetch(match.value)] } { query: normalized, queryFeatures: features(query_features), indexedCount: @labels.size, total: matches.size, rebuilt: rebuilt, matches: matches.first(100).map do |match| candidate = @extractor.features(match.value) { value: @labels.fetch(match.value), normalized: match.value, score: match.score, shared: (query_features & candidate).size, features: features(candidate), } end, } end def self.features(set) set.map { |feature| { text: feature.ngram, occurrence: feature.index } } end end