package RPS::Sale::Match; use strict; #use warnings; use Data::Dumper; use String::Similarity; use Encode qw(encode_utf8); use lib '/app/tools/sale_import/lib'; use lib '/app/tools/rps/lib/'; use lib '/app/tools/data_classes/lib'; use lib '/app/tools/common/lib'; use Common::Util qw(clean clean_name_catalog word_containment); use Common::RSDB; use Common::RSApp; use Common::Log; use Client::Service; #use Product::ProductAlbum; # require'd in MakeMatch use RPS::DB::Item::Product; use RPS::DB::Item::MediaType; use RPS::Sale::Match::Result; use RPS::File::Sale; use constant NP => 0; use constant NOMATCH => 1; use constant ISMATCH => 2; use constant SUGGEST => 'suggest'; use constant MATCH => 'match'; use constant STATUS_MATCH => 1; use constant STATUS_NOMATCH => 2; use constant STATUS_MULTIMATCH => 3; use constant STATUS_MAPPED => 4; use constant STATUS_UNRECOVERABLE => 5; use constant STATUS_AUTO_MAPPED => 6; use constant STATUS_BATCH_MAPPED => 7; use constant STATUS_DONT_MATCH => 8; use constant LIMIT_SEARCH => 100; use base 'Sale::Match'; sub errstr { my $self = shift; return $self->{errstr}; } sub SearchCatalog { my $self = shift; my(%in) = @_; my $search_type = lc $in{type}; my $keyword = $in{keyword} || $in{keywords}; my $product_type_id = _translate_product_type($in{product_type}); my $limit = (exists $in{limit} && $in{limit}) ? $in{limit} : Sale::Match::LIMIT_SEARCH; my $data = (exists $in{data}) ? $in{data} : undef; my $media_type = $data->{media_type}; # for backwards compatibility (temporary) if (exists $in{all} && defined $in{all}) { $search_type = 'all'; $keyword = $in{all}; } #my @search_fields = qw(artist album track isrc upc); my @search_fields = qw(track album artist isrc upc); my %search_types_allowed = map {$_ => 1} ('smart', 'all', @search_fields); unless (defined $search_type && exists $search_types_allowed{$search_type}) { $self->{errstr} = "You must specify a valid type parameter: " . join(', ', keys %search_types_allowed); return undef; } unless (defined $keyword && length(clean_name_catalog($keyword)) >= 3 ) { $self->{errstr} = "You must specify a keyword value with at least three letters and/or numbers"; return undef; } unless (defined $product_type_id && $product_type_id =~ /^\d+$/) { $self->{errstr} = "You must specify a valid product_type value"; return undef; } if (defined $limit && ($limit !~ /^\d+$/ || $limit < 1) ) { $self->{errstr} = "The value for 'limit' must be an integer greater than zero"; return undef; } my $keyword_clean = clean_name_catalog($keyword); my $smart_key_found; if ($search_type eq 'smart') { unless (defined $data && ref($data) eq 'HASH') { $self->{errstr} = "You must specify a data hashref for a smart search"; return undef; } my $keyword_match = $keyword; $keyword_match =~ s/([^\w\s])/\\$1/g; foreach my $field (@search_fields) { next unless (exists $data->{$field} && defined $data->{$field}); next if length($data->{$field}) < 3; # try to see if our search keyword is a substring of any of the product data fields if ($data->{$field} =~ /$keyword_match/i) { $smart_key_found = $field; last; } # next try to match the "clean" version my $data_field_clean = clean_name_catalog($data->{$field}); if (length($keyword_clean) >= 3 && length($data_field_clean) >= 3 && $data_field_clean =~ /$keyword_clean/) { $smart_key_found = $field; last; } } # if we couldn't limit the search to a specific field, revert to searching by type $search_type = $smart_key_found ? $smart_key_found : $product_type_id == RPS::DB::Item::Product::kProductTypeDigitalTrack ? 'track' : 'album'; } Sale::Match::DEBUG && print STDERR "PRODUCT_SEARCH: search_type = $search_type\n"; my @search_products = (); if ($search_type eq 'all') { my $albumMatches = $self->_search_album($keyword, $product_type_id, $media_type); my $artistMatches = $self->_search_artist($keyword, $product_type_id, $media_type); my $trackMatches = $self->_search_track($keyword, $product_type_id, $media_type); my $upcMatches = $self->_search_upc($keyword, $product_type_id, $media_type); my $isrcMatches = $self->_search_isrc($keyword, $product_type_id, $media_type); if ($product_type_id == RPS::DB::Item::Product::kProductTypeDigitalTrack) { push(@search_products, @$trackMatches) if $trackMatches; push(@search_products, @$isrcMatches) if $isrcMatches; push(@search_products, @$albumMatches) if $albumMatches; push(@search_products, @$upcMatches) if $upcMatches; push(@search_products, @$artistMatches) if $artistMatches; } else { push(@search_products, @$albumMatches) if $albumMatches; push(@search_products, @$upcMatches) if $upcMatches; push(@search_products, @$artistMatches) if $artistMatches; push(@search_products, @$trackMatches) if $trackMatches; push(@search_products, @$isrcMatches) if $isrcMatches; } } elsif ($search_type eq 'album') { my $albumMatches = $self->_search_album($keyword, $product_type_id, $media_type); push(@search_products, @$albumMatches) if $albumMatches; } elsif ($search_type eq 'artist') { my $artistMatches = $self->_search_artist($keyword, $product_type_id, $media_type); push(@search_products, @$artistMatches) if $artistMatches; } elsif ($search_type eq 'track') { my $trackMatches = $self->_search_track($keyword, $product_type_id, $media_type); push(@search_products, @$trackMatches) if $trackMatches; } elsif ($search_type eq 'upc') { my $upcMatches = $self->_search_upc($keyword, $product_type_id, $media_type); push(@search_products, @$upcMatches) if $upcMatches; } elsif ($search_type eq 'isrc') { my $isrcMatches = $self->_search_isrc($keyword, $product_type_id, $media_type); push(@search_products, @$isrcMatches) if $isrcMatches; } my $detail = {}; foreach my $product_id (@search_products) { $detail->{$product_id} = { level => 60, type => $search_type . '_search', }; } return new RPS::Sale::Match::Result( products => [], num_products => 0, import_status => undef, map_id => undef, rec_products => [], search_products => \@search_products, detail => $detail ); } sub FindBestMatch { my $self = shift; my(%in) = @_; my $skip_rec = (exists $in{skip_rec} && $in{skip_rec}) ? 1 : 0; my $map_only = $in{map_only}; my $product_type_id = _translate_product_type($in{data}{product_type}); Sale::Match::DEBUG && print STDERR 'FindBestMatch data: ' . Dumper($in{data}) . "\n"; # check to see if we have a previous map entry for this input data # if we do, then we can short-circuit the normal search # !!! This is already abstract enough. - JPK # if (my $result = $self->_search_product_input_map($in{data})) { return $result if defined $result; } return undef if $map_only; # Call the appropriate _get_*_matches method to retrieve the set of possible matches. # # We'll use this data to construct the Match::Result object. # my $matches = {}; if ($product_type_id == RPS::DB::Item::Product::kProductTypeDigitalTrack) { $matches = $self->_get_track_matches($in{data}, $skip_rec); } else { $matches = $self->_get_album_matches($in{data}, $product_type_id, $skip_rec); } my @matchedProducts; my @suggestedProducts; # Iterate through all the matches, sorted by weight. # Note that if we are not returning any 'recommendations', then we only want # to return the results that all share the highest weight. # my $max_weight = 0; # !!! Sort by weight and album_score. foreach my $id ($self->_sort_by_weight_and_scores($matches)) { if ($skip_rec) { if ($matches->{$id}{weight} < $max_weight || $matches->{$id}{status} eq SUGGEST) { delete $matches->{$id}; next; } push @matchedProducts, $id; $max_weight = $matches->{$id}{weight}; } else { if ($matches->{$id}{status} eq SUGGEST) { push @suggestedProducts, $id; delete $matches->{id}; } else { push @matchedProducts, $id; } } } # JPK - If we're looking for exact matches (i.e. not recommendations) and # we have more than one possible match, try to winnow it down further. # -> We might want to put this in a seperate method, so we can override it # to adjust our strategy for different customers. # But, for now, we'll use the original artist name filter mechanism. # my $num_products = scalar @matchedProducts; if ($skip_rec && $num_products > 1) { # JPK - Rather than call this filter method (which is probably not going to help a whole hell of a lot) # we could either a) adjust the weights in the truth table to give artist MATCH greater impact, or # b) we could record the trust table 'vector', and remove results where ARTIST was not a MATCH. # # !!! Going to comment this out, since so far it doesn't seem to actually _help_ us. # @matchedProducts = $self->_filter_by_artist_name($in{data}{artist}, \@matchedProducts, $matches) if $in{data}{artist}; # $num_products = scalar @matchedProducts; } # !!! A couple of ramifications of this code 'merging': # - We're returning more status types for 'recommended' matches. # Need to make sure this doesn't freak out the Matching Tool. # - We'll also return something in the 'detail' field of the Result when # doing a recommended matched: This shouldn't cause any problems, but # worth keeping in mind. # my $import_status = $num_products == 1 ? STATUS_MATCH : $num_products == 0 ? STATUS_NOMATCH : $num_products > 1 ? STATUS_MULTIMATCH : ''; my $result = RPS::Sale::Match::Result->new ( products => \@matchedProducts, num_products => $num_products, import_status => $import_status, map_id => undef, rec_products => \@suggestedProducts, search_products => undef, detail => $matches, ); return $result; } sub RemoveMatch { my $self = shift; my %in = @_; my $map_id = (exists $in{map_id}) ? $in{map_id} : undef; unless (defined $map_id) { $self->{errstr} = "You must specify a map_id param"; return undef; } my $sql = "DELETE FROM product_input_map WHERE map_id = ?"; my $sth = $self->dbh()->prepare($sql); unless ($sth) { $self->errstr = $self->dbh()->errstr || "Can't prepare SQL"; return undef; } unless ($sth->execute($map_id)) { $self->{errstr} = $sth->errstr; return undef; } return 1; } sub MakeMatch { my $self = shift; my %in = @_; return undef unless (ref($in{data}) eq 'HASH'); unless ($in{product_id}) { $self->{errstr} = 'You must specify a product_id param'; return undef; } my $data = $in{data}; my $product_type_id = _translate_product_type($data->{product_type}); my $product_id = $in{product_id}; my $allow_reuse = 0; # If our input data does NOT meet a certain threshold, # do NOT allow our map entry to be re-used automatically in the future # (use it ONLY for this one-time manual product match) if ($product_type_id == RPS::DB::Item::Product::kProductTypeDigitalTrack) { $allow_reuse = 1 if ($data->{track} && ($data->{artist} || $data->{isrc}) && $data->{service_id}); } elsif ($product_type_id == RPS::DB::Item::Product::kProductTypeDigital) { $allow_reuse = 1 if ($data->{album} && ($data->{artist} || $data->{upc}) && $data->{service_id}); } else # physical { require Product::ProductAlbum; my $product = Product::ProductAlbum->new(product_id => $product_id); if ($data->{upc}) { if ($data->{upc} =~ /^\d{10}\d*$/ && $data->{upc} !~ /0{6}/) { if ($product_type_id == $product->ProductTypeID) { $allow_reuse = 1; } } } elsif ($data->{album}) { $allow_reuse = 1 if ($self->_cleanAlbum($data) eq $product->AlbumNameClean); } } my $match_md5 = $self->_match_md5(data => $data) || return undef; my $sql = qq{ INSERT INTO product_input_map (product_id, product_type, media_type, match_md5_new, upc, isrc, artist_name, album_name, track_name, track_num, service_id, service_product_id, allow_reuse, date_created) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, now()) }; my $sth = $self->dbh()->prepare($sql); unless ($sth) { $self->{errstr} = $self->dbh()->errstr || "Can't prepare SQL"; return undef; } unless ($sth->execute( $product_id, $data->{product_type}, $data->{media_type}, $match_md5, $data->{upc}, $data->{isrc}, $data->{artist}, $data->{album}, $data->{track}, $data->{track_num}, $data->{service_id}, $data->{service_product_id}, $allow_reuse )) { $self->{errstr} = $sth->errstr; return undef; } # my $map_id = $self->rsdb->LastInsertID(); my $map_id = Common::RSApp::GetClientDB()->LastInsertID(); unless ($map_id) { $self->{errstr} = "Could not get the last inserted map_id"; return undef; } return $map_id; } # private methods sub _translate_product_type { my $type = uc $_[0]; return $type eq RPS::File::Sale::TYPE_ALBUM ? RPS::DB::Item::Product::kProductTypeDigital : $type eq RPS::File::Sale::TYPE_TRACK ? RPS::DB::Item::Product::kProductTypeDigitalTrack : $type eq RPS::File::Sale::TYPE_LP ? RPS::DB::Item::Product::kProductTypeLP : $type eq RPS::File::Sale::TYPE_LP5 ? RPS::DB::Item::Product::kProductTypeLP5 : $type eq RPS::File::Sale::TYPE_CD ? RPS::DB::Item::Product::kProductTypeCD : $type eq RPS::File::Sale::TYPE_VHS ? RPS::DB::Item::Product::kProductTypeVHS : $type eq RPS::File::Sale::TYPE_CASS ? RPS::DB::Item::Product::kProductTypeCass : $type eq RPS::File::Sale::TYPE_EP ? RPS::DB::Item::Product::kProductTypeEP : $type eq RPS::File::Sale::TYPE_DVD ? RPS::DB::Item::Product::kProductTypeDVD : $type eq RPS::File::Sale::TYPE_BLURAY ? RPS::DB::Item::Product::kProductTypeBluRay : $type eq RPS::File::Sale::TYPE_CAS_SIN ? RPS::DB::Item::Product::kProductTypeCassSingle : $type eq RPS::File::Sale::TYPE_CD_SIN ? RPS::DB::Item::Product::kProductTypeCDSingle : $type eq RPS::File::Sale::TYPE_DVD_CD_SET ? RPS::DB::Item::Product::kProductTypeDVDCDSet : $type eq RPS::File::Sale::TYPE_DBL_CD ? RPS::DB::Item::Product::kProductTypeDblCD : ''; } sub _do_list_query { my $self = shift; Sale::Match::DEBUG && print STDERR 'list query: ' . join(',', @_) . "\n"; # my $sth = $self->rsdb->DBH->prepare(shift); #die join(',', @_); # !!! This violates encapsulation... my $sth = Common::RSApp::GetClientDB()->DBH->prepare(shift); $sth->execute(@_); if ($sth->rows) { my @list = map { @{$_} } @{ $sth->fetchall_arrayref([0]) }; return \@list; } return undef; } sub _search_product_input_map { my $self = shift; my $data = shift; ## try new maps my $match_md5 = $self->_match_md5(data => $data) || return undef; my ($map_id, $product_id, $dont_match) = $self->_lookup_product_map(md5 => $match_md5, product_type => $data->{product_type}); unless ($map_id && defined $product_id) { ## no match found using new-style maps so check old ones $match_md5 = $self->_match_md5(data=> $data, old_map => 1) || return undef; ($map_id, $product_id, $dont_match) = $self->_lookup_product_map(md5 => $match_md5, product_type => $data->{product_type}, old_map => 1); } return undef unless ($map_id && defined $product_id); my $import_status = $dont_match ? STATUS_DONT_MATCH : $self->{map_type}; return RPS::Sale::Match::Result->new( products => [$product_id], num_products => 1, import_status => $import_status, map_id => $map_id, rec_products => [], search_products => [], detail => { $product_id => { level => 100, type => 'map', }, } ); } sub _match_md5 { my $self = shift; my %in = @_; my $data = $in{data}; my $old_style = $in{old_map} || 0;; my @match_fields = (); if (uc $data->{product_type} eq 'A') { @match_fields = qw(product_type upc artist album); push(@match_fields, qw(service_id service_product_id)) unless $old_style; } elsif (uc $data->{product_type} eq 'T') { @match_fields = qw(product_type isrc upc artist album track); push(@match_fields, qw(track_num service_id service_product_id)) unless $old_style; } elsif (uc $data->{product_type} =~ m/^(\d|C|K|M|D|B|P|U)$/) # physical { @match_fields = qw(product_type upc artist album); push(@match_fields, qw(service_id service_product_id)) unless $old_style; } else { print STDERR "unknown product type: '$data->{product_type}'\n"; return undef; } push(@match_fields, 'media_type') if ($data->{media_type} != RPS::DB::Item::MediaType::kMediaTypeAudio); # copy and adjust the values my %args = ( product_type => uc $data->{product_type}, media_type => $data->{media_type}, isrc => uc $data->{isrc}, upc => uc $data->{upc}, artist => lc $data->{artist}, album => lc $data->{album}, track => lc $data->{track}, track_num => lc $data->{track_num}, service_id => lc $data->{service_id}, service_product_id => lc $data->{service_product_id}, ); my $md5_string = ''; map { $md5_string .= "$_=$args{$_};" } @match_fields; $md5_string =~ s/;$//; Sale::Match::DEBUG && print STDERR "md5 arg: $md5_string\n"; # We're going to "encode" the input MD5 string as UTF8, but strictly for the purpose # of clearing the UTF8 flag for the field, which makes Digest::MD5::md5_base64() happy # (as it doesn't operate on wide characters, i.e. scalars with the UTF8 flag set). # We _don't_ care if the string even contains a valid (or correct) UTF8 sequences, but # since encode_utf8() doesn't do any data conversion other than clearing the flag, this # seems the most conservative thing to do. The alternative would have been to use # Encode::_utf8_off() for this purpose, but given that this is embroiled in warnings # regarding it's internal nature and being subject to change, the choice of encode_utf8() # seems prudent return Digest::MD5::md5_base64(encode_utf8($md5_string)); } sub _lookup_product_map { my $self = shift; my %in = @_; my $md5_field = 'match_md5_new'; $md5_field = 'match_md5' if $in{old_map}; my $sql = qq{SELECT map_id, product_id, dont_match FROM product_input_map WHERE $md5_field = ? AND product_type = ? AND allow_reuse = 1}; Sale::Match::DEBUG && print STDERR "sql: $sql\nparams: $in{md5}, $in{product_type}\n"; my $sth = $self->dbh()->prepare($sql); unless ($sth) { $self->{errstr} = $self->dbh()->errstr || "Can't prepare SQL: $sql"; return undef; } unless ($sth->execute($in{md5}, $in{product_type})) { $self->{errstr} = $sth->errstr; return undef; } return ($sth->fetchrow_array()); } sub _best_recommended_search { my $self = shift; my $data = shift; my $prodTypeID = shift; #select track_name from track where match(track_name) against ('Napster Presents - A Conversation With Nas') limit 30; my ($sql, $match, $limit); my @params = (); if ($prodTypeID == RPS::DB::Item::Product::kProductTypeDigitalTrack && defined $data->{track}) { push @params, $data->{track}; my $matchAgainst; my $orderBy = 'track_score DESC'; if (defined($data->{artist} && $data->{artist} !~ m/various/i)) { $sql = "SELECT product_id, MATCH(track.title) AGAINST(?) as track_score," . " MATCH(artist.name) AGAINST(?) as artist_score" . " FROM artist, track, product WHERE" . " track.artist_id=artist.artist_id AND "; # $match = 'artist.name,track.title'; $matchAgainst = "MATCH(track.title) AGAINST(?) AND MATCH(artist.name) AGAINST(?)"; $orderBy .= ', artist_score DESC'; push(@params, $data->{artist}, $data->{track}, $data->{artist}); } else { $sql = "SELECT product_id, MATCH(track.title) AGAINST(?) as track_score FROM track, product WHERE"; $matchAgainst = "MATCH(track.title) AGAINST (?)"; push @params, $data->{track}; } $sql .= " product.asset_id=track.track_id" . " AND product.product_type_id=$prodTypeID" . " AND $matchAgainst" . " AND track.media_type=?" . " ORDER BY $orderBy" . " LIMIT " . Sale::Match::LIMIT_SEARCH; push @params, $data->{media_type}; } elsif (defined $data->{album}) { $match = 'product.title'; $sql = "SELECT product_id, MATCH($match) AGAINST(?) as album_score" . " FROM album, product" . " WHERE product.asset_id=album.album_id" . " AND product.product_type_id = $prodTypeID" . " AND MATCH($match) AGAINST(?)" . " ORDER BY album_score DESC" . " LIMIT 30"; push(@params, $data->{album}, $data->{album}); } else { return (); } my $sth = $self->dbh()->prepare($sql) || print STDERR "rec search error: " . $self->dbh->errstr; Sale::Match::DEBUG && print STDERR "rec sql: $sql\n" . join(',', @params, "\n"); $sth->execute(@params) or print STDERR "rec search error: " . $sth->errstr; # my $sql = "SELECT product_id FROM product_catalog_extract WHERE vendor_id in($types) and product_type=? and match($match) against(?) limit $limit"; # Sale::Match::DEBUG && print STDERR join(',', $sql, @$media_types, $data->{product_type}, $value, "\n"); # $sth->execute(@$media_types, $data->{product_type}, $value) or print STDERR "rec search error: " . $sth->errstr; my @productList = map { $_->[0] } @{ $sth->fetchall_arrayref([0]) }; return @productList; } sub _get_album_matches { my ($self, $data, $prodTypeID, $skipRec) = @_; my $albumClean = $self->_cleanAlbum($data); my $artistClean = $self->_cleanArtist($data); my $upc = $data->{upc}; return undef unless ($upc || $albumClean); my $dbo = Common::RSApp::GetClientDB(); my @select; my @where; my @join; push @select, "product.product_id AS album_product_id"; push @select, "product.upc_ean AS upc"; push @select, "product.upc_alt AS upc_alt"; push @select, "artist.name AS artist_name"; push @select, "artist.name_clean AS artist_name_clean"; push @select, "product.title AS album_title"; push @select, "product.title_clean AS album_title_clean"; push @select, "IF (product_type_id <> $prodTypeID, 1, 0) AS suggestion"; push @join, "LEFT JOIN album ON (album.album_id = product.asset_id)"; push @join, "LEFT JOIN artist ON (artist.artist_id = album.artist_id)"; if ($prodTypeID eq RPS::DB::Item::Product::kProductTypeDigital || $skipRec) { push @where, "product.product_type_id = $prodTypeID"; } else { push @where, "product.product_type_id NOT IN (" . RPS::DB::Item::Product::kProductTypeDigital . ", " . RPS::DB::Item::Product::kProductTypeDigitalTrack . ")"; } my @subSql; if ($albumClean) { # push @subWhere, "album.title_clean = ".$dbo->DBQuote($albumClean); my $sql = "SELECT " . join(',', @select) . " FROM product " . join(' ', @join) . " WHERE " . join(' AND ', @where) . ' AND product.title_clean = '.$dbo->DBQuote($albumClean); push @subSql, $sql; } if ($upc) { # !!! This double-sided regex slows the query down considerably. # !!! It might actually be faster to try padding '0's on the UPC, and union together a few different versions. # !!! I'm thinking that if the incoming UPC has 11 or 12 characters, we'll do another _exact_ query # !!! using a 0-padded upc (padded to 13 characters). # my $upcSearch = '%'.$upc.'%'; my $upcSearch = "$upc%"; # push @subWhere, "product.upc_ean like ".$dbo->DBQuote($upcSearch); my $sql = "SELECT " . join(',', @select) . " FROM product " . join(' ', @join) . " WHERE " . join(' AND ', @where) . " AND product.upc_ean like ".$dbo->DBQuote($upcSearch); push @subSql, $sql; my $upcSize = length $upc; if (11 == $upcSize || 12 == $upcSize) { my $padded = '0' x (13 - $upcSize); $padded .= $upc; my $sql = "SELECT " . join(',', @select) . " FROM product " . join(' ', @join) . " WHERE " . join(' AND ', @where) . " AND product.upc_ean = ".$dbo->DBQuote($padded); push @subSql, $sql; } } # push @where, '(' . join(' OR ', @subWhere) . ')'; # my $sql = "SELECT " . join(',', @select) . " FROM product " . join(' ', @join) . " WHERE " . join(' AND ', @where); my $sql = join(' UNION ', @subSql); # Now make this initial query, and iterate over the results to find good matches. # my %matches; my $sth = $dbo->DoCmd($sql); my $foundMatchesFlag = 0; while (my $hr = $sth->fetchrow_hashref()) { my ($matchStatus, $matchWeight) = $self->_scoreAlbumRow(row => $hr, data => $data, album_clean => $albumClean, artist_clean => $artistClean); next unless defined $matchStatus; $matchStatus = SUGGEST if ($hr->{suggestion}); $foundMatchesFlag++ if (MATCH eq $matchStatus); $matches{$hr->{album_product_id}} = { data => $hr, # <-- we'll see how this goes, since I've changed the query... I don't yet know who examines this hash... status => $matchStatus, weight => $matchWeight, }; } # Need an album name to attempt a fuzzy search. # if (! $skipRec && ! $foundMatchesFlag && $albumClean) { my @orderBy; my @matchAgainst; # We want all the same columns as in the initial query. Really, we're just replacing the 'where' clause and adding in # the MATCH stuff to the select clause. # # Need to transform the clean name by replacing underscores with spaces. Otherwise the MATCH AGAINST # algorithm will treat the whole name as one word... # my $albumCleanWithSpaces = $albumClean; $albumCleanWithSpaces =~ s/_/ /g; my $albumRaw = $data->{album}; my $artistRaw = $data->{artist}; push @select, "MATCH(product.title) AGAINST (" . $dbo->DBQuote($albumCleanWithSpaces) . ") as album_score"; push @matchAgainst, "MATCH(product.title) AGAINST(".$dbo->DBQuote($albumCleanWithSpaces).")"; push @orderBy, "album_score DESC"; if ($artistClean && $artistClean !~ m/various/) { push @select, "MATCH(artist.name) AGAINST (".$dbo->DBQuote($artistRaw).") as artist_score"; push @matchAgainst, "MATCH(artist.name) AGAINST (".$dbo->DBQuote($artistRaw).")"; push @orderBy, "artist_score DESC"; } my @searchWhere; if ($prodTypeID eq RPS::DB::Item::Product::kProductTypeDigital) { push @searchWhere, "product.product_type_id = $prodTypeID"; } else { push @searchWhere, "product.product_type_id NOT IN (" . RPS::DB::Item::Product::kProductTypeDigital . ", " . RPS::DB::Item::Product::kProductTypeDigitalTrack . ")"; } push @searchWhere, join(' AND ', @matchAgainst); my $sql = "SELECT " . join(',', @select) . " FROM product " . join(' ', @join) . " WHERE " . join(' AND ', @searchWhere) . " ORDER BY " . join(',', @orderBy) . " LIMIT " . Sale::Match::LIMIT_SEARCH; my $sth = $dbo->DoCmd($sql); while (my $hr = $sth->fetchrow_hashref()) { # Ignore duplicates. # next if defined $matches{$hr->{album_product_id}}; my ($matchStatus, $matchWeight) = $self->_scoreAlbumRow(row => $hr, data => $data, album_clean => $albumClean, artist_clean => $artistClean); next unless defined $matchStatus; $matchStatus = SUGGEST if ($hr->{suggestion}); $matches{$hr->{album_product_id}} = { data => $hr, # <-- we'll see how this goes, since I've changed the query... I don't yet know who examines this hash... status => $matchStatus, weight => $matchWeight, }; } } return \%matches; } sub _get_track_matches { my ($self, $data, $skipRec) = @_; # First, let's create clean or 'normalized' versions of the parameters # we might use to construct the query. # my $trackClean = $self->_cleanTrack($data); my $albumClean = $self->_cleanAlbum($data); my $artistClean = $self->_cleanArtist($data); my $isrc = $self->_cleanISRC($data); return undef unless ($isrc or $trackClean); my $dbo = Common::RSApp::GetClientDB(); my @select; my @where; my @subWhere; my @join; push @select, "product.product_id AS track_product_id"; push @select, "master.isrc AS isrc"; push @select, "artist.name AS artist_name"; push @select, "artist.name_clean AS artist_name_clean"; push @select, "track.title AS track_title"; push @select, "track.title_clean AS track_title_clean"; push @select, "album_product.title AS album_title"; push @select, "album_product.title_clean AS album_title_clean"; push @select, "album_product.upc_ean AS upc"; push @select, "album_product.upc_alt AS upc_alt"; push @join, "LEFT JOIN track ON (product.asset_id = track.track_id)"; push @join, "LEFT JOIN master ON (master.master_id = track.master_id)"; push @join, "LEFT JOIN artist ON (artist.artist_id = track.artist_id)"; push @join, "LEFT JOIN album ON (album.album_id = track.album_id)"; push @join, "LEFT JOIN product AS album_product ON" . " (album_product.product_id = product.parent_product_id)"; push @where, "product.product_type_id=" . RPS::DB::Item::Product::kProductTypeDigitalTrack; push @where, "track.media_type='".$data->{media_type}."'"; # If we have both a clean track name AND an isrc, then we want to essentially do an 'OR' type query. # Sadly, this causes trouble: The mysql query optimizer will throw up it's hands, so to speak, and do # a full table scan of product, which slows us down considerably. # # So there are two approaches we can take: # 1) We can make two queries: one with the title_clean WHERE clause, and one with the ISRC WHERE clause, and then # consolidate the results here in code. This is what the current code does. And, hey, it works. # # 2) We can do a UNION query. This really is the same thing as option 1, except that the database does all the consolidation # for us. It might be slightly faster, too. # # So I'm going to go with option 2. Makes for a beefy looking query, but it does run fast. # my @subSql; if (defined $trackClean) { my $sql = "SELECT " . join(',', @select) . " FROM product " . join(' ', @join) . " WHERE " . join(' AND ', @where) . ' AND track.title_clean='.$dbo->DBQuote($trackClean); push @subSql, $sql; } if (defined $isrc) { my $sql = "SELECT " . join(',', @select) . " FROM product " . join(' ', @join) . " WHERE " . join(' AND ', @where) . ' AND master.isrc='.$dbo->DBQuote($isrc); push @subSql, $sql; } my $sql = join(' UNION ', @subSql); # Now make this initial query, and iterate over the results to find good matches. # my %matches; my $sth = $dbo->DoCmd($sql); # If we get any results with a status of ISMATCH, set this flag. # We'll use this later to decide whether to do a 'fuzzy' search. # !!! What I am not sure about yet is whether we want to skip the fuzzy # !!! search in this case or not. There _could_ be a better match waiting out # !!! there. Then again, it's significantly slower to do that second query. # !!! Perhaps we may want a 'weight threshold' - Don't bother looking if the weight # !!! is 3 or more, for example. # my $foundMatchesFlag = 0; while (my $hr = $sth->fetchrow_hashref()) { my ($matchStatus, $matchWeight) = $self->_scoreTrackRow(row => $hr, data => $data, album_clean => $albumClean, track_clean => $trackClean, artist_clean => $artistClean); next unless defined $matchStatus; $foundMatchesFlag++ if (MATCH eq $matchStatus); $matches{$hr->{track_product_id}} = { data => $hr, # <-- we'll see how this goes, since I've changed the query... I don't yet know who examines this hash... status => $matchStatus, weight => $matchWeight, }; } # Need a track name to attempt a fuzzy search. # if (! $skipRec && ! $foundMatchesFlag && $trackClean) { my @orderBy; my @matchAgainst; # We want all the same columns as in the initial query. Really, we're just replacing the 'where' clause and adding in # the MATCH stuff to the select clause. # # !!! What I would _really_ like to do is a full text index search against _normalized_ titles. # !!! essentially the clean name with '_' replaced by ' ' (so the MATCH AGAINST algorithm will work). # !!! I think that might help the MATCH AGAINST algorithm work better when we have 'wide' characters. # !!! But, that's yet another column, so probably won't go there unless absolutely necessary. # my $trackCleanWithSpaces = $trackClean; $trackCleanWithSpaces =~ s/_/ /g; my $trackRaw = $data->{track}; my $artistRaw = $data->{artist}; push @select, "MATCH(track.title) AGAINST (".$dbo->DBQuote($trackCleanWithSpaces).") as track_score"; push @matchAgainst, "MATCH(track.title) AGAINST(".$dbo->DBQuote($trackCleanWithSpaces).")"; push @orderBy, "track_score DESC"; if ($artistClean && $artistClean !~ m/various/) { push @select, "MATCH(artist.name) AGAINST (".$dbo->DBQuote($artistRaw).") as artist_score"; push @matchAgainst, "MATCH(artist.name) AGAINST (".$dbo->DBQuote($artistRaw).")"; push @orderBy, "artist_score DESC"; } my @searchWhere; push @searchWhere, "product.product_type_id=" . RPS::DB::Item::Product::kProductTypeDigitalTrack; #push @searchWhere, "track.media_type=".$dbo->DBQuote($data->{media_type}); push @searchWhere, join(' AND ', @matchAgainst); my $sql = "SELECT " . join(',', @select) . " FROM product " . join(' ', @join) . " WHERE " . join(' AND ', @searchWhere) . " ORDER BY " . join(',', @orderBy) . " LIMIT " . Sale::Match::LIMIT_SEARCH; my $sth = $dbo->DoCmd($sql); while (my $hr = $sth->fetchrow_hashref()) { # Ignore duplicates. # next if defined $matches{$hr->{track_product_id}}; my ($matchStatus, $matchWeight) = $self->_scoreTrackRow(row => $hr, data => $data, album_clean => $albumClean, track_clean => $trackClean, artist_clean => $artistClean); next unless defined $matchStatus; $matches{$hr->{track_product_id}} = { data => $hr, # <-- we'll see how this goes, since I've changed the query... I don't yet know who examines this hash... status => SUGGEST, # let's force these to suggestions as they're not necessarily quality matches # (see 'string containment' in testTrackTitle()) # ...and not to mention we're returning recommendations here anyway weight => $matchWeight, }; } } return \%matches; } sub _scoreAlbumRow { my ($self, %args) = @_; my $hr = $args{row}; my $data = $args{data}; my $albumClean = $args{album_clean}; my $artistClean = $args{artist_clean}; my ($upcFlag, $upcExactFlag) = $self->_testUPC(row => $hr, data => $data); my $albumFlag = $self->_testAlbumTitle(row => $hr, data => $data, clean => $albumClean); my $artistFlag = $self->_testArtistName(row => $hr, data => $data, clean => $artistClean); my $tt = $self->_get_album_tt(); return (undef, undef) unless exists $tt->{$upcFlag}{$albumFlag}{$artistFlag}; my $matchStatus = $tt->{$upcFlag}{$albumFlag}{$artistFlag}{status}; my $matchWeight = $tt->{$upcFlag}{$albumFlag}{$artistFlag}{weight}; # Increase the weight of matches where the UPC was 'exact'. # $matchWeight++ if $upcExactFlag; return ($matchStatus, $matchWeight); } sub _scoreTrackRow { my ($self, %args) = @_; my $hr = $args{row}; my $data = $args{data}; my $albumClean = $args{album_clean}; my $trackClean = $args{track_clean}; my $artistClean = $args{artist_clean}; my ($upcFlag, $upcExactFlag) = $self->_testUPC(row => $hr, data => $data); my $albumFlag = $self->_testAlbumTitle(row => $hr, data => $data, clean => $albumClean); my $trackFlag = $self->_testTrackTitle(row => $hr, data => $data, clean => $trackClean); my $isrcFlag = $self->_testISRC(row => $hr, data => $data); my $artistFlag = $self->_testArtistName(row => $hr, data => $data, clean => $artistClean); my $tt = $self->_get_track_tt(); #Common::Log::Print("_scoreTrackRow: row:", $hr, " data:", $data, "upcFlag: $upcFlag, albumFlag: $albumFlag, isrcFlag: $isrcFlag, trackFlag: $trackFlag, artistFlag: $artistFlag"); return (undef, undef) unless exists $tt->{$upcFlag}{$albumFlag}{$isrcFlag}{$trackFlag}{$artistFlag}; my $matchStatus = $tt->{$upcFlag}{$albumFlag}{$isrcFlag}{$trackFlag}{$artistFlag}{status}; my $matchWeight = $tt->{$upcFlag}{$albumFlag}{$isrcFlag}{$trackFlag}{$artistFlag}{weight}; # Increase the weight of matches where the UPC was 'exact'. # $matchWeight++ if $upcExactFlag; #Common::Log::Print("RETURNING matchStatus $matchStatus matchWeight $matchWeight \n"); return ($matchStatus, $matchWeight); } sub _get_by_id { my ($self, $sql, $params, $result) = @_; my $sth; if (ref($params) eq 'ARRAY') { # $sth = $self->rsdb()->DoCmdWithPlaceholders($sql, $params); $sth = Common::RSApp::GetClientDB()->DoCmdWithPlaceholders($sql, $params); } else { $sth = Common::RSApp::GetClientDB()->DoCmd($sql); } return unless $sth->rows(); while (my $row = $sth->fetchrow_hashref()) { $result->{$row->{primary_id}} = $row unless (exists $result->{ $row->{primary_id} }); } return $sth->rows(); } sub _search_album { my ($self, $keyword, $prodType, $mediaType) = @_; my $keywordClean = clean_name_catalog($keyword); my ($sql); my $whereClean; my $whereMatch; if ($prodType eq RPS::DB::Item::Product::kProductTypeDigitalTrack) { $sql = "SELECT p.product_id FROM product p" . " JOIN product AS album_product ON (album_product.product_id = p.parent_product_id AND album_product.product_type_id = 3)" . " WHERE p.product_type_id = $prodType"; $whereClean = " AND ? = album_product.title_clean"; $whereMatch = " AND MATCH(album_product.title) AGAINST (?)"; } else { $sql = "SELECT product.product_id FROM product"; if ($prodType eq RPS::DB::Item::Product::kProductTypeDigital) { $sql .= " WHERE product.product_type_id = $prodType"; } else { $sql .= " WHERE product.product_type_id NOT IN (" . RPS::DB::Item::Product::kProductTypeDigital . ", " . RPS::DB::Item::Product::kProductTypeDigitalTrack . ")"; } $whereClean = " AND product.title_clean=?"; $whereMatch = " AND MATCH(product.title) AGAINST (?)"; } return $self->_search_catalog( sql => $sql, conditions => [$whereClean, $whereMatch], args => [[$keywordClean], [$keyword]], ); } sub _search_artist { my ($self, $keyword, $prodType, $mediaType) = @_; my $keywordClean = clean_name_catalog($keyword); my $sql; if ($prodType == RPS::DB::Item::Product::kProductTypeDigitalTrack) { $sql = "SELECT product.product_id FROM track, product, artist" . " WHERE product.product_type_id = $prodType" . " AND track.track_id=product.asset_id" . " AND track.artist_id=artist.artist_id"; } else { $sql = "SELECT product.product_id FROM artist, album, product" . " WHERE album.album_id=product.asset_id" . " AND artist.artist_id=album.artist_id"; if ($prodType eq RPS::DB::Item::Product::kProductTypeDigital) { $sql .= " AND product.product_type_id = $prodType"; } else { $sql .= " AND product.product_type_id NOT IN (" . RPS::DB::Item::Product::kProductTypeDigital . ", " . RPS::DB::Item::Product::kProductTypeDigitalTrack . ")"; } } my $whereClean = " AND artist.name_clean=?"; my $whereMatch = " AND MATCH(artist.name) AGAINST (?)"; return $self->_search_catalog( sql => $sql, conditions => [$whereClean, $whereMatch], args => [[$keywordClean], [$keyword]], ); } sub _search_track { my ($self, $keyword, $prodType, $mediaType) = @_; my $keywordClean = clean_name_catalog($keyword); if ($prodType != RPS::DB::Item::Product::kProductTypeDigitalTrack) { $self->errstr('abort: track search on a non-track product'); return undef; } my $sql = "SELECT product.product_id FROM track, product" . " WHERE product.product_type_id = $prodType" . " AND track.track_id=product.asset_id"; my $whereClean = " AND track.title_clean=?"; my $whereMatch = " AND MATCH(track.title) AGAINST (?)"; return $self->_search_catalog( sql => $sql, conditions => [$whereClean, $whereMatch], args => [[$keywordClean], [$keyword]], ); } sub _search_upc { my ($self, $keyword, $prodType, $mediaType) = @_; my ($prodList, $sql); if ($prodType == RPS::DB::Item::Product::kProductTypeDigitalTrack) { $sql = "SELECT album.album_id FROM album JOIN product ON album.album_id=product.asset_id" . " WHERE product.product_type_id = ".RPS::DB::Item::Product::kProductTypeDigital . " AND (product.upc_ean=? OR product.upc_alt=?" . " OR product.upc_ean like ? OR product.upc_alt like ?)" . " LIMIT 20"; my $keywordWild = '%' . $keyword . '%'; if (my $albumList = $self->_do_list_query($sql, $keyword, $keyword, $keywordWild, $keywordWild)) { my $albumIDs = join(',', @$albumList); $sql = "SELECT product.product_id FROM track, product" . " WHERE product.product_type_id = $prodType" . " AND track.track_id=product.asset_id" . " AND track.album_id in($albumIDs)" . " LIMIT " . Sale::Match::LIMIT_SEARCH; $prodList = $self->_do_list_query($sql); } } else { $sql = "SELECT product.product_id FROM album, product" . " WHERE album.album_id=product.asset_id" . " AND (product.upc_ean=? OR product.upc_alt=?" . " OR product.upc_ean like ? OR product.upc_alt like ?)"; if ($prodType eq RPS::DB::Item::Product::kProductTypeDigital) { $sql .= " AND product.product_type_id = $prodType"; } else { $sql .= " AND product.product_type_id NOT IN (" . RPS::DB::Item::Product::kProductTypeDigital . ", " . RPS::DB::Item::Product::kProductTypeDigitalTrack . ")"; } $sql .= " LIMIT " . Sale::Match::LIMIT_SEARCH; my $keywordWild = '%' . $keyword . '%'; $prodList = $self->_do_list_query($sql, $keyword, $keyword, $keywordWild, $keywordWild); } return $prodList; } sub _search_isrc { my ($self, $keyword, $prodType, $mediaType) = @_; if ($prodType != RPS::DB::Item::Product::kProductTypeDigitalTrack) { $self->errstr('abort: isrc search on a non-track product'); return undef; } my $sql = "SELECT product.product_id FROM master, track, product" . " WHERE product.product_type_id = $prodType" . " AND track.track_id=product.asset_id" . " AND master.master_id=track.master_id" . " AND (master.isrc=? or master.isrc like ? or track.custom_1=?)" . " LIMIT " . Sale::Match::LIMIT_SEARCH; return $self->_do_list_query($sql, $keyword, '%'.$keyword.'%', $keyword); } sub _search_catalog { my $self = shift; my %params = @_; my $limit = $params{limit} || Sale::Match::LIMIT_SEARCH; my %prodIdSeen = (); my @result = (); my $i = 0; foreach my $condition(@{ $params{conditions} }) { my $sql = join(' ', $params{sql}, $condition, 'LIMIT', $limit); my $prodList = $self->_do_list_query($sql, @{ $params{args}->[$i] }); if (ref($prodList) eq 'ARRAY') { map { push(@result, $_) unless (exists $prodIdSeen{$_}); $prodIdSeen{$_} = 1; } @$prodList; } last if (scalar @result > 10); $i++; } return scalar @result ? \@result : undef; } sub _get_album_tt { my $self = shift; if (! defined $self->{album_tt}{ISMATCH()}) { my $tt = {}; # UPC ALBUM ARTIST $tt->{NP()} {ISMATCH()} {NP()} = { status => MATCH, weight => 1 }; $tt->{NP()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 1 }; $tt->{NP()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NP()} {NP()} = { status => MATCH, weight => 1 }; $tt->{ISMATCH()} {NP()} {ISMATCH()} = { status => MATCH, weight => 1 }; $tt->{ISMATCH()} {NP()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NOMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NOMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {ISMATCH()} {NP()} = { status => MATCH, weight => 1 }; $tt->{ISMATCH()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {ISMATCH()} {NOMATCH()} = { status => MATCH, weight => 1 }; $tt->{NOMATCH()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 1 }; $tt->{NP()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 1 }; $self->{album_tt} = $tt; } return $self->{album_tt}; } sub _get_track_tt { my ($self) = @_; # !!! We have some pointless premature initialization in the base class, which causes this # !!! simple check to fail... # # if (! $self->{track_tt}) if (! defined $self->{track_tt}{ISMATCH()}) { my $tt = {}; # UPC Album ISRC Track Artist $tt->{NP()} {NP()} {NP()} {ISMATCH()} {NP()} = { status => MATCH, weight => 1 }; $tt->{NP()} {NP()} {NP()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 1 }; $tt->{NP()} {NP()} {NP()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NP()} {NOMATCH()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NP()} {NOMATCH()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NP()} {NOMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NP()} {ISMATCH()} {NP()} {NP()} = { status => MATCH, weight => 1 }; $tt->{NP()} {NP()} {ISMATCH()} {NP()} {ISMATCH()} = { status => MATCH, weight => 1 }; $tt->{NP()} {NP()} {ISMATCH()} {NP()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NP()} {ISMATCH()} {NOMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NP()} {ISMATCH()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NP()} {ISMATCH()} {NOMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NP()} {ISMATCH()} {ISMATCH()} {NP()} = { status => MATCH, weight => 1 }; $tt->{NP()} {NP()} {ISMATCH()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 2 }; # !!! I think this should weigh more. $tt->{NP()} {NP()} {ISMATCH()} {ISMATCH()} {NOMATCH()} = { status => MATCH, weight => 1 }; # !!! <- this matches current logic better. $tt->{NP()} {NOMATCH()} {NP()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {NP()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {NP()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {NOMATCH()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {NOMATCH()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {NOMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {ISMATCH()} {NP()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {ISMATCH()} {NP()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {ISMATCH()} {NP()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {ISMATCH()} {NOMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {ISMATCH()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {ISMATCH()} {NOMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {ISMATCH()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {ISMATCH()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {NOMATCH()} {ISMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {ISMATCH()} {NP()} {NOMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {ISMATCH()} {NP()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {ISMATCH()} {NP()} {NOMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {ISMATCH()} {NP()} {ISMATCH()} {NP()} = { status => MATCH, weight => 1 }; $tt->{NP()} {ISMATCH()} {NP()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 1 }; $tt->{NP()} {ISMATCH()} {NP()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {ISMATCH()} {NOMATCH()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {ISMATCH()} {NOMATCH()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {ISMATCH()} {NOMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {ISMATCH()} {ISMATCH()} {NP()} {NP()} = { status => MATCH, weight => 1 }; $tt->{NP()} {ISMATCH()} {ISMATCH()} {NP()} {ISMATCH()} = { status => MATCH, weight => 1 }; $tt->{NP()} {ISMATCH()} {ISMATCH()} {NP()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {ISMATCH()} {ISMATCH()} {NOMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {ISMATCH()} {ISMATCH()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {ISMATCH()} {ISMATCH()} {NOMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NP()} {ISMATCH()} {ISMATCH()} {ISMATCH()} {NP()} = { status => MATCH, weight => 1 }; $tt->{NP()} {ISMATCH()} {ISMATCH()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 1 }; $tt->{NP()} {ISMATCH()} {ISMATCH()} {ISMATCH()} {NOMATCH()} = { status => MATCH, weight => 1 }; # <- matches current logic $tt->{NOMATCH()} {NP()} {NP()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NP()} {NP()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NP()} {NP()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NP()} {NOMATCH()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NP()} {NOMATCH()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NP()} {NOMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NP()} {ISMATCH()} {NOMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NP()} {ISMATCH()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NP()} {ISMATCH()} {NOMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NP()} {ISMATCH()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NP()} {ISMATCH()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NP()} {ISMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {NP()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {NP()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {NP()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {NOMATCH()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {NOMATCH()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {NOMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {ISMATCH()} {NP()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {ISMATCH()} {NP()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {ISMATCH()} {NP()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {ISMATCH()} {NOMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {ISMATCH()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {ISMATCH()} {NOMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {ISMATCH()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {ISMATCH()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {NOMATCH()} {ISMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {NP()} {NOMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {NP()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {NP()} {NOMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {NP()} {ISMATCH()} {NP()} = { status => MATCH, weight => 1 }; $tt->{NOMATCH()} {ISMATCH()} {NP()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 1 }; $tt->{NOMATCH()} {ISMATCH()} {NP()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {NOMATCH()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {NOMATCH()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {NOMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {ISMATCH()} {NP()} {NP()} = { status => MATCH, weight => 1 }; $tt->{NOMATCH()} {ISMATCH()} {ISMATCH()} {NP()} {ISMATCH()} = { status => MATCH, weight => 1 }; $tt->{NOMATCH()} {ISMATCH()} {ISMATCH()} {NP()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {ISMATCH()} {NOMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {ISMATCH()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {ISMATCH()} {NOMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{NOMATCH()} {ISMATCH()} {ISMATCH()} {ISMATCH()} {NP()} = { status => MATCH, weight => 1 }; $tt->{NOMATCH()} {ISMATCH()} {ISMATCH()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 1 }; $tt->{NOMATCH()} {ISMATCH()} {ISMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NP()} {NP()} {ISMATCH()} {NP()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {NP()} {NP()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 3 }; # !!! Making this worth more. $tt->{ISMATCH()} {NP()} {NP()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; # !!! Not sure if this should be a MATCH $tt->{ISMATCH()} {NP()} {NOMATCH()} {ISMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NP()} {NOMATCH()} {ISMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NP()} {NOMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NP()} {ISMATCH()} {NP()} {NP()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {NP()} {ISMATCH()} {NP()} {ISMATCH()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {NP()} {ISMATCH()} {NP()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NP()} {ISMATCH()} {NOMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NP()} {ISMATCH()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NP()} {ISMATCH()} {NOMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NP()} {ISMATCH()} {ISMATCH()} {NP()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {NP()} {ISMATCH()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 3 }; # <- Increasing the weight. $tt->{ISMATCH()} {NP()} {ISMATCH()} {ISMATCH()} {NOMATCH()} = { status => MATCH, weight => 1 }; # <- matches original logic better $tt->{ISMATCH()} {NOMATCH()} {NP()} {ISMATCH()} {NP()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {NOMATCH()} {NP()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {NOMATCH()} {NP()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NOMATCH()} {NOMATCH()} {ISMATCH()} {NP()} = { status => MATCH, weight => 1 }; $tt->{ISMATCH()} {NOMATCH()} {NOMATCH()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 1 }; $tt->{ISMATCH()} {NOMATCH()} {NOMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NOMATCH()} {ISMATCH()} {NP()} {NP()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {NOMATCH()} {ISMATCH()} {NP()} {ISMATCH()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {NOMATCH()} {ISMATCH()} {NP()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NOMATCH()} {ISMATCH()} {NOMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NOMATCH()} {ISMATCH()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NOMATCH()} {ISMATCH()} {NOMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {NOMATCH()} {ISMATCH()} {ISMATCH()} {NP()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {NOMATCH()} {ISMATCH()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {NOMATCH()} {ISMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {ISMATCH()} {NP()} {ISMATCH()} {NP()} = { status => MATCH, weight => 3 }; $tt->{ISMATCH()} {ISMATCH()} {NP()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 3 }; $tt->{ISMATCH()} {ISMATCH()} {NP()} {ISMATCH()} {NOMATCH()} = { status => MATCH, weight => 1 }; # <- matches original logic better $tt->{ISMATCH()} {ISMATCH()} {NOMATCH()} {ISMATCH()} {NP()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {ISMATCH()} {NOMATCH()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 2 }; $tt->{ISMATCH()} {ISMATCH()} {NOMATCH()} {ISMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {ISMATCH()} {ISMATCH()} {NP()} {NP()} = { status => MATCH, weight => 3 }; $tt->{ISMATCH()} {ISMATCH()} {ISMATCH()} {NP()} {ISMATCH()} = { status => MATCH, weight => 3 }; $tt->{ISMATCH()} {ISMATCH()} {ISMATCH()} {NP()} {NOMATCH()} = { status => MATCH, weight => 1 }; # <- matches original logic better $tt->{ISMATCH()} {ISMATCH()} {ISMATCH()} {NOMATCH()} {NP()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {ISMATCH()} {ISMATCH()} {NOMATCH()} {ISMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {ISMATCH()} {ISMATCH()} {NOMATCH()} {NOMATCH()} = { status => SUGGEST, weight => 0 }; $tt->{ISMATCH()} {ISMATCH()} {ISMATCH()} {ISMATCH()} {NP()} = { status => MATCH, weight => 3 }; $tt->{ISMATCH()} {ISMATCH()} {ISMATCH()} {ISMATCH()} {ISMATCH()} = { status => MATCH, weight => 4 }; $tt->{ISMATCH()} {ISMATCH()} {ISMATCH()} {ISMATCH()} {NOMATCH()} = { status => MATCH, weight => 2 }; # <- matches original logic better. $self->{track_tt} = $tt; } return $self->{track_tt}; } sub _sort_by_weight_and_scores { my ($self, $matches) = @_; # Let's sort by all the scores as well. # return sort { my $w = $matches->{$b}{weight} <=> $matches->{$a}{weight}; if (! $w) { $w = $matches->{$b}{data}{track_score} <=> $matches->{$a}{data}{track_score}; } if (! $w) { $w = $matches->{$b}{data}{album_score} <=> $matches->{$a}{data}{album_score}; } if (! $w) { $w = $matches->{$b}{data}{artist_score} <=> $matches->{$a}{data}{artist_score}; } $w; } keys %$matches; } # !!! I am not totally sold on this. # !!! Seems like this is redundant somehow (although we are checking the unmodified artist name not the clean name). # sub _filter_by_artist_name { my ($self, $sale_artist, $matchedProductIDs, $matches) = @_; my @matchList; foreach my $productID (@$matchedProductIDs) { my $match = $matches->{$productID}; if (word_containment($sale_artist, $match->{data}{artist_name})) { push @matchList, $productID; } else { delete $matches->{$productID}; } } return @matchList; } sub _filter_by_release_date { my ($self, $matches) = @_; my $min_date = 99991231; while (my ($id, $attr) = each %$matches) { my $date = $attr->{data}{release_date}; $date =~ s/-//g; # now it's just a number $min_date = $date if ($date < $min_date); } Sale::Match::DEBUG && print STDERR "min release date is $min_date\n"; while (my ($id, $attr) = each %$matches) { my $date = $attr->{data}{release_date}; $date =~ s/-//g; delete $matches->{$id} unless ($date == $min_date); } } #my @recommended = keys %$matches ? $self->_sort_by_weight($matches) : #$self->_sort_suggestions($matches, $in{data}{format}) : sub _sort_suggestions { my ($self, $sale_data, $matches) = @_; my @sorted_list = (); my @artist_nomatch = (); foreach my $id($self->_sort_by_weight_and_scores($matches)) { unless (word_containment($sale_data->{artist}, $matches->{$id}{data}{artist_name})) { push(@artist_nomatch, $id); next; } push(@sorted_list, $id); } push(@sorted_list, @artist_nomatch) if @artist_nomatch; return @sorted_list; } # !!! These methods are old-school, and not necessary. # #sub rsdb { # my $self = shift; # my %in = @_; # # if (defined $self->{rsdb}) { # return $self->{rsdb}; # } # # unless (defined $self->{client_id}) { # if (exists $in{client_id} && $in{client_id} =~ /^\d+$/) { # $self->{client_id} = $in{client_id}; # } else { # $self->{errstr} = "client_id not defined"; # # # that's ok, Common::RSDB will get it from $ENV # # return undef; # } # } # # my $rsdb = new Common::RSDB(client_id => $self->{client_id}, dbi_attr => {RaiseError => 1}); # # unless (defined $rsdb) { # die "rsdb not defined!\n"; # } # # $self->{rsdb} = $rsdb; # # return $self->{rsdb}; #} # sub dbh { my $self = shift; # my %in = @_; # # return $self->rsdb(%in)->DBH(); return Common::RSApp::GetClientDB()->DBH(); } sub _cleanTrack { my ($self, $data) = @_; my $track = $data->{track}; return undef unless defined $track; return clean_name_catalog($track); } sub _cleanAlbum { my ($self, $data) = @_; my $album = $data->{album}; return undef unless defined $album; # JPK - iTunes will add some stuff, sometimes, to the album title. Remove that. # if ($data->{service_id} && Client::Service::DSP_ITUNES == $data->{service_id}) { $album =~ s/- (ep|single)$//i; # !!! this is the wrong direction. the _catalog_ has 'digitally' in it, the sale does not # !!! Whatever we do will need to be EMG only # $album =~ s/\(digitally remastered\)$/\(remastered\)/i; } return clean_name_catalog($album); } sub _cleanArtist { my ($self, $data) = @_; my $artist = $data->{artist}; return undef unless defined $artist; return clean_name_catalog($artist); } sub _cleanISRC { my ($self, $data) = @_; my $isrc = $data->{isrc}; # Default is a noop return $isrc; } sub _testUPC { my ($self, %args) = @_; my $upc_flag = NP; my $row = $args{row}; my $data = $args{data}; # !!! This seems extremely unlikely, but if OUR data doesn't include a UPC, then we # obviously cannot match against it... # # return (NP, 0) unless $row->{upc}; # return (NP, 0) unless $data->{upc}; # !!! So the intent here is to gather up all the wackiness related to comparing # !!! UPCs, and hide them away in this method. # !!! - The 'UPC' can be formatted in various ways. # !!! - It can appear in multiple fields in the incoming data. # # !!! So to start out, I'm going to copy all the stuff we currently do straight into here. # !!! Some of it looks redundant to me, though, so there may be some logical consolidation. # my $upc = ($data->{upc} && length $data->{upc} == 13 && substr($data->{upc},0,1) == 0) ? substr($data->{upc},1,10) : ($data->{upc} && length $data->{upc} > 11) ? substr($data->{upc},0,10) : ($data->{vendor_id} && length $data->{vendor_id} > 11) ? substr($data->{vendor_id},0,10) : undef; my $upc_exact = ($data->{upc} && length $data->{upc} == 13 && substr($data->{upc},0,1) == 0) ? substr($data->{upc},1) : $data->{upc}; my $upc_opt = length $data->{client_product_id} > 10 ? $data->{client_product_id} : length $data->{service_product_id} > 10 ? $data->{service_product_id} : undef; $upc_opt = substr($upc_opt,0,10) if ($upc_opt && length $upc_opt < 13); # 'meta' in this case refers to 'metadata', i.e. our internal data. # my $metaUPC = $row->{upc}; my $metaUPCAlt = $row->{upc_alt}; # Let's also try to match by stripping off the leading zero of 13 digit UPCs. my $metaUPCEAN = $metaUPC; if ($metaUPCEAN && length $metaUPCEAN == 13 && substr($metaUPCEAN,0,1) == 0) { $metaUPCEAN = substr($metaUPCEAN,1); } #Common::Log::Print("upc:$upc upc_exact:$upc_exact upc_opt:$upc_opt metaUPC:$metaUPC metaUPCAlt:$metaUPCAlt"); if (defined $upc) { # !!! None of these will catch the case where the metadata is missing the leading 0, and the sale data has it. # !!! I'm going to add in a reverse regex that should catch that. # $upc_flag = ($metaUPC =~ /^$upc/ or (length $metaUPC >= 11 && length $upc_exact >= 11 && $upc_exact =~ /$metaUPC/) or $metaUPCEAN =~ /^$upc/ or $metaUPCAlt eq $data->{upc}) ? ISMATCH : NOMATCH; } if ($upc_flag != ISMATCH and defined $upc_opt) { # only override to ISMATCH otherwise leave as it was (NOMATCH or NP) # !!! This assumes the UPC portion appears first. # !!! But I've been seeing it come LAST, i.e. 'ISRC_UPC' # $upc_flag = ISMATCH if ($metaUPC =~ /^$upc_opt/); # So this will return true if the metaUPC appears someplace if ($metaUPC) { $upc_flag = ISMATCH if ($upc_opt =~ /$metaUPC/); } } my $exactFlag = 0; # Increase the weight if we have an 'exact match'. # if (defined $upc_exact and ($metaUPC eq $upc_exact or $metaUPCAlt eq $upc_exact or $metaUPCEAN eq $upc_exact)) { $exactFlag = 1; } return ($upc_flag, $exactFlag); } # By default we'll establish '95% similar' as our closeness threshold. # This is pretty tight, corresponding to a 1 character difference in a 20 character string. # We'll wrap a method around this so we can override it if necessary. # sub _albumScoreMatchThreshold { return '0.95'; } sub _trackScoreMatchThreshold { return '0.95'; } sub _artistScoreMatchThreshold { return '0.95'; } sub _testAlbumTitle { my ($self, %args) = @_; my ($album_flag, $weight); my $row = $args{row}; my $data = $args{data}; my $cleanTitle = $args{clean}; return NP unless defined $cleanTitle; my $metaCleanTitle = $row->{album_title_clean}; # JPK - There are a variety of strategies we can employ here to 'score' the title. # I feel like ultimately the String::Similarity algorithm may be just the thing. # However, we may need to consider the string word-by-word, since many titles are # quite short. The point, though, is not to generate any false positives, while # still producing a weight value that is useful for sorting results. # $weight = String::Similarity::similarity($metaCleanTitle, $cleanTitle); if ($weight >= $self->_albumScoreMatchThreshold()) { $album_flag = ISMATCH; } else { $album_flag = NOMATCH; } return $album_flag; } sub _testTrackTitle { my ($self, %args) = @_; my ($track_flag, $weight); my $row = $args{row}; my $data = $args{data}; my $cleanTitle = $args{clean}; return NP unless defined $cleanTitle; $track_flag = NOMATCH; my $metaCleanTitle = $row->{track_title_clean}; $weight = String::Similarity::similarity($metaCleanTitle, $cleanTitle); if ($weight >= $self->_trackScoreMatchThreshold()) { $track_flag = ISMATCH; } elsif (length $cleanTitle > 2 && length $metaCleanTitle > 2) { # JPK - We'll use the 'string containment' heuristic from the old code. # In the old code we only did this when the ISRC matched... For now # I want to just do it in all cases, and we'll see what happens. # Note that I don't see the point of filtering out the underscores - The new # clean name mechanism should make that unnecessary. # # 2 characters seems pretty small... # # SAH - consider tightening it up to a bit, maybe check to make sure the shortest # is at least half the length of the longest, something like: # abs($lenTitle - $lenMeta) < $lenMax / 2 # # we force these to suggestions when used in fallback !$skipRec mode in # _get_track_matches() anyway, so those are presented to users, limiting "risk" # if (index($cleanTitle, $metaCleanTitle) > -1 || index($metaCleanTitle, $cleanTitle) > -1) { $track_flag = ISMATCH; } } return $track_flag; } sub _testISRC { my ($self, %args) = @_; my $row = $args{row}; my $data = $args{data}; return NP unless $row->{isrc}; return NP unless $data->{isrc}; my $isrc_flag = lc($row->{isrc}) eq lc($data->{isrc}) ? ISMATCH : NOMATCH; return $isrc_flag; } sub _testArtistName { my ($self, %args) = @_; my $row = $args{row}; my $data = $args{data}; my $cleanName = $args{clean}; return NP unless defined $cleanName; # The current artist name scheme uses word containment. # We could continue with that strategy, or we could do something more akin # to the similarity test. # # I'm actually inclined to combine the two: We can use 'similarity' to generate the weight, # but set the flag based on word_containment. # # my $metaClean = $row->{track_artist_clean}; my $metaClean = $row->{artist_name_clean}; #Common::Log::Print("_testArtistName: cleanName='$cleanName', metaClean='$metaClean'"); my $flag = NOMATCH; my $weight = String::Similarity::similarity($cleanName, $metaClean); #Common::Log::Print(" similarity weight: $weight"); if ($weight >= $self->_artistScoreMatchThreshold()) { $flag = ISMATCH; } else { # We'll leave the weight untouched, but let's try one more time using # the slightly looser 'word_containment' algorithm to see if this is a match. # my $cleanNameSpaces = $cleanName; $cleanNameSpaces =~ s/_/ /g; my $metaCleanSpaces = $metaClean; $metaCleanSpaces =~ s/_/ /g; #Common::Log::Print(" word_containment: cleanNameSpaces = $cleanNameSpaces, metaCleanNameSpaces = $metaCleanSpaces"); # !!! Passing the 'skip normalization flag' as true, because these names are already normalized. if (word_containment($cleanNameSpaces, $metaCleanSpaces, 1)) { #Common::Log::Print(" word_containment returned TRUE"); $flag = ISMATCH; } } return $flag; } 1;