From: Rogan Hamby <rhamby@esilibrary.com>
Date: Wed, 30 Oct 2019 17:44:17 +0000 (-0400)
Subject: adding changes to fingerprinter supplied by Blake Henderson blake@mobiusconsortium... 
X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=commitdiff_plain;h=775c66de891990851afcde5fbec9f6a8c6d004c3

adding changes to fingerprinter supplied by Blake Henderson blake@mobiusconsortium.org makes 2 changes 1)  match_fingerprints wouldn't continue to the next line if the first line
failed to match 2) The fingerprinter needed a different sort. I used a 17 character
zero-padded ID from the bib ID as the sort.
---

diff --git a/fingerprinter b/fingerprinter
index f388174..8442970 100755
--- a/fingerprinter
+++ b/fingerprinter
@@ -75,7 +75,7 @@ for my $file (@ARGV) {
 
         # if everything looks good, score it and dump fingerprints
         score_marc($marc, $record);
-        dump_fingerprints($marc);
+        dump_fingerprints_score_id($marc);
         $scount++; progress_ticker();
     }
 }
@@ -102,6 +102,8 @@ sub populate_marc {
 
     # date1, date2
     my $my_008 = $record->field('008');
+    my @my_007 = $record->field('007');
+    my $my_006 = $record->field('006');
     $marc{tag008} = $my_008->as_string() if ($my_008);
     if (defined $marc{tag008}) {
         unless (length $marc{tag008} == 40) {
@@ -123,12 +125,35 @@ sub populate_marc {
             }
         }
     }
+    $marc{tag006} = $my_006->as_string() if ($my_006);
+    $marc{tag007} = \@my_007 if (@my_007);
+    $marc{audioformat}='';
+    $marc{videoformat}='';
+    foreach(@my_007)
+    {
+        if(substr($_->data(),0,1) eq 's' && $marc{audioformat} eq '')
+        {
+            $marc{audioformat} = substr($_->data(),3,1) unless (length $_->data() < 4);
+        }
+        elsif(substr($_->data(),0,1) eq 'v' && $marc{videoformat} eq '')
+        {
+            $marc{videoformat} = substr($_->data(),4,1) unless (length $_->data() < 5);
+        }
+    }
+    #print "$marc{audioformat}\n";
+    #print "$marc{videoformat}\n";
 
     # item_form
+    $marc{item_form}='';
     if ( $marc{record_type} =~ /[gkroef]/ ) { # MAP, VIS
-        $marc{item_form} = substr($marc{tag008},29,1) if ($marc{tag008});
+        $marc{item_form} = substr($marc{tag008},29,1) if ($marc{tag008} && (length $marc{tag008} > 29 ));
     } else {
-        $marc{item_form} = substr($marc{tag008},23,1) if ($marc{tag008});
+        $marc{item_form} = substr($marc{tag008},23,1) if ($marc{tag008} && (length $marc{tag008} > 23 ));
+    }
+    #fall through to 006 if 008 doesn't have info for item form
+    if ($marc{item_form} eq '|')
+    {
+        $marc{item_form} = substr($marc{tag006},6,1) if ($marc{tag006} && (length $marc{tag006} > 6 ));
     }
 
     # isbns
@@ -419,6 +444,227 @@ sub dump_fingerprints {
     }
 }
 
+sub dump_fingerprints_score_id {
+    my ($marc) = @_;
+
+    if ($conf->{fingerprints}{baseline}) {
+        print OF join("\t", sortvalfromid($marc->{id}),"json", $marc->{id}, 'baseline',
+                      $marc->{item_form}, $marc->{date1}, $marc->{record_type},
+                      $marc->{bib_lvl},$marc->{audioformat},$marc->{videoformat}, $marc->{title}), "\n";
+    }
+
+    if ($conf->{fingerprints}{oclc} and scalar @{$marc->{oclc} }) {
+        for (@{$marc->{oclc} }) {
+            print OF join("\t", sortvalfromid($marc->{id}),"json", $marc->{id}, "oclc",
+                          $marc->{item_form}, $marc->{date1},
+                          $marc->{record_type}, $marc->{bib_lvl},$marc->{audioformat},$marc->{videoformat},
+                          $marc->{title}, $_, "\n");
+        }
+    }
+
+    if ($conf->{fingerprints}{koha_bib_id} and exists $marc->{koha_bib_id}) {
+        print OF join("\t", sortvalfromid($marc->{id}),"json", $marc->{id}, "z_koha_bib_id",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type},
+                      $marc->{bib_lvl},$marc->{audioformat},$marc->{videoformat}, $marc->{title},
+                      $marc->{koha_bib_id}), "\n";
+    }
+
+    if ($conf->{fingerprints}{isbn}) {
+        if ((scalar @{ $marc->{isbns} } > 0) and $marc->{pages}) {
+            foreach my $isbn ( @{ $marc->{isbns}} ) {
+                print OF join("\t", sortvalfromid($marc->{id}),"json", $marc->{id}, "isbn",
+                              $marc->{item_form}, $marc->{date1},
+                              $marc->{record_type},
+                              $marc->{bib_lvl},$marc->{audioformat},$marc->{videoformat}, $marc->{title},
+                              $isbn, $marc->{pages}), "\n";
+            }
+        }
+    }
+
+    if ($conf->{fingerprints}{edition} and $marc->{edition} and $marc->{author}) {
+        print OF join("\t", sortvalfromid($marc->{id}),"json", $marc->{id}, "edition",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},$marc->{audioformat},$marc->{videoformat},
+                      $marc->{title}, $marc->{author}, $marc->{edition}), "\n";
+    }
+
+    if ($conf->{fingerprints}{issn} and $marc->{issn}) {
+        print OF join("\t", sortvalfromid($marc->{id}),"json", $marc->{id}, "issn",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},$marc->{audioformat},$marc->{videoformat},
+                      $marc->{title}, $marc->{issn}), "\n";
+    }
+
+    if ($conf->{fingerprints}{lccn} and $marc->{lccn}) {
+        print OF join("\t", sortvalfromid($marc->{id}),"json", $marc->{id}, "lccn",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},$marc->{audioformat},$marc->{videoformat},
+                      $marc->{title}, $marc->{lccn}) ,"\n";
+    }
+
+    if ($conf->{fingerprints}{accomp} and $marc->{accomp}) {
+        print OF join("\t", sortvalfromid($marc->{id}),"json", $marc->{id}, "accomp",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},$marc->{audioformat},$marc->{videoformat},
+                      $marc->{title}, $marc->{accomp}) ,"\n";
+    }
+
+    if ($conf->{fingerprints}{authpub} and $marc->{author} and
+        $marc->{publisher} and $marc->{pubyear} and $marc->{pages}) {
+        print OF join("\t", sortvalfromid($marc->{id}),"json", $marc->{id}, "authpub",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},$marc->{audioformat},$marc->{videoformat},
+                      $marc->{title}, $marc->{author},
+                      $marc->{publisher}, $marc->{pubyear},
+                      $marc->{pages}), "\n";
+    }
+}
+
+sub sortvalfromid
+{
+    my $sortval = shift;
+    while(length($sortval)<17)
+    {
+        $sortval = '0'.$sortval;
+    }
+    return $sortval;
+}
+
+sub dump_fingerprints_hash_score {
+    my ($marc) = @_;
+
+    if ($conf->{fingerprints}{baseline}) {
+        my $string = join("", 'baseline',
+                      $marc->{item_form}, $marc->{date1}, $marc->{record_type},
+                      $marc->{bib_lvl}, $marc->{title});
+        $string =~ s/[^A-Za-z0-9]//g;
+        $string = sha1_base64($string);
+        print OF join("\t", $string,"json", $marc->{id}, 'baseline',
+                      $marc->{item_form}, $marc->{date1}, $marc->{record_type},
+                      $marc->{bib_lvl}, $marc->{title}), "\n";
+    }
+
+    if ($conf->{fingerprints}{oclc} and scalar @{$marc->{oclc} }) {
+        for (@{$marc->{oclc} }) {
+            my $string = join("", "oclc",
+                          $marc->{item_form}, $marc->{date1},
+                          $marc->{record_type}, $marc->{bib_lvl},
+                          $marc->{title}, $_);
+            $string =~ s/[^A-Za-z0-9]//g;
+            $string = sha1_base64($string);
+            print OF join("\t", $string,"json", $marc->{id}, "oclc",
+                          $marc->{item_form}, $marc->{date1},
+                          $marc->{record_type}, $marc->{bib_lvl},
+                          $marc->{title}, $_, "\n");
+        }
+    }
+
+    if ($conf->{fingerprints}{koha_bib_id} and exists $marc->{koha_bib_id}) {
+        my $string = join("", "z_koha_bib_id",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type},
+                      $marc->{bib_lvl}, $marc->{title},
+                      $marc->{koha_bib_id});
+        $string =~ s/[^A-Za-z0-9]//g;
+        $string = sha1_base64($string);
+        print OF join("\t", $string,"json", $marc->{id}, "z_koha_bib_id",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type},
+                      $marc->{bib_lvl}, $marc->{title},
+                      $marc->{koha_bib_id}), "\n";
+    }
+
+    if ($conf->{fingerprints}{isbn}) {
+        if ((scalar @{ $marc->{isbns} } > 0) and $marc->{pages}) {
+            foreach my $isbn ( @{ $marc->{isbns}} ) {
+                my $string = join("", "isbn",
+                              $marc->{item_form}, $marc->{date1},
+                              $marc->{record_type},
+                              $marc->{bib_lvl}, $marc->{title},
+                              $isbn, $marc->{pages});
+                $string =~ s/[^A-Za-z0-9]//g;
+                $string = sha1_base64($string);
+                print OF join("\t", $string,"json", $marc->{id}, "isbn",
+                              $marc->{item_form}, $marc->{date1},
+                              $marc->{record_type},
+                              $marc->{bib_lvl}, $marc->{title},
+                              $isbn, $marc->{pages}), "\n";
+            }
+        }
+    }
+
+    if ($conf->{fingerprints}{edition} and $marc->{edition} and $marc->{author}) {
+        my $string = join("", "edition",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{author}, $marc->{edition});
+        $string =~ s/[^A-Za-z0-9]//g;
+        $string = sha1_base64($string);
+        print OF join("\t", $string,"json", $marc->{id}, "edition",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{author}, $marc->{edition}), "\n";
+    }
+
+    if ($conf->{fingerprints}{issn} and $marc->{issn}) {
+        my $string = join("", "issn",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{issn});
+        $string =~ s/[^A-Za-z0-9]//g;
+        $string = sha1_base64($string);
+        print OF join("\t", $string,"json", $marc->{id}, "issn",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{issn}), "\n";
+    }
+
+    if ($conf->{fingerprints}{lccn} and $marc->{lccn}) {
+        my $string = join("", "lccn",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{lccn});
+        $string =~ s/[^A-Za-z0-9]//g;
+        $string = sha1_base64($string);
+        print OF join("\t", $string,"json", $marc->{id}, "lccn",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{lccn}) ,"\n";
+    }
+
+    if ($conf->{fingerprints}{accomp} and $marc->{accomp}) {
+        my $string = join("", "accomp",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{accomp});
+        $string =~ s/[^A-Za-z0-9]//g;
+        $string = sha1_base64($string);
+        print OF join("\t", $string,"json", $marc->{id}, "accomp",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{accomp}) ,"\n";
+    }
+
+    if ($conf->{fingerprints}{authpub} and $marc->{author} and
+        $marc->{publisher} and $marc->{pubyear} and $marc->{pages}) {
+        my $string = join("", "authpub",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{author},
+                      $marc->{publisher}, $marc->{pubyear},
+                      $marc->{pages});
+        $string =~ s/[^A-Za-z0-9]//g;
+        $string = sha1_base64($string);
+        print OF join("\t", $string,"json", $marc->{id}, "authpub",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{author},
+                      $marc->{publisher}, $marc->{pubyear},
+                      $marc->{pages}), "\n";
+    }
+}
+
 
 
 =head2 dump_exception
diff --git a/match_fingerprints b/match_fingerprints
index 4c77480..8f0d485 100755
--- a/match_fingerprints
+++ b/match_fingerprints
@@ -62,7 +62,7 @@ sub populate_fingerprint {
     $fp{sha1}    = sha1_base64($stripped);
 
     # make sure file is sorted properly
-    if ($lastscore and ($fp{compact} > $lastscore)) {
+    if ($lastscore and ($fp{compact} < $lastscore)) {
         print "Input file is sorted improperly or unsorted.\n";
         die "Sort descending (sort -r) and rerun this script.\n";
     }
@@ -79,7 +79,7 @@ sub rank_fingerprint {
     my $id   = $fp->{id};
 
     # only process records which haven't already been seen
-    unless (defined $seen{$id}) {
+    #unless (defined $seen{$id}) {
         unless (defined $fps{$sha1}) {
             # haven't seen this fp before. create a new listref to hold subs
             # and stow the hash of the fingerprint that we're lead of
@@ -90,7 +90,7 @@ sub rank_fingerprint {
             push @{ $fps{$sha1} }, $id;
         }
         $seen{$id} = 1;
-    }
+    #}
 }
 
 
@@ -107,16 +107,38 @@ sub dump_records {
     for my $rec (@recs) {
         for ( @{ $fps{ $rec->{sha1} } } ) {
             # check for dupes and die if they exist
-            die "Collision: dupe sub record $_\n" if $subs{$_};
-            $subs{$_} = 1;
-            die "Collision: lead in sub list ", $rec->{id}, "\n"
-              if $subs{ $rec->{id} };
-
-            # we don't want subs below threshold
-            next if ($_ < $conf->{threshold});
-
-            # still here? output.
-            print OUT $rec->{id}, "\t$_\n"
+            #die "Collision: dupe sub record $_\n" if $subs{$_};
+            if( not exists ($subs{$_}) ) {
+                #die "Collision: lead in sub list ", $rec->{id}, "\n"
+                #  if $subs{ $rec->{id} };
+                my $lead = $rec->{id};
+
+                # we don't need to match onto itself
+                next if ($lead eq $_);
+
+                if ($subs{ $rec->{id} })
+                {
+                    print "moving ".$lead." to ".$subs{$rec->{id}}."\n";
+                    $lead = $subs{$rec->{id}};
+                }
+
+                if($_ eq '1459051')
+                {
+                    print "looking at ".$rec->{sha1};
+                }
+
+                # we don't want subs below threshold
+                next if ($_ < $conf->{threshold});
+
+                # we don't need to match onto itself
+                next if ($lead eq $_);
+
+                # record this sub having this leader
+                $subs{$_} = $lead;
+
+                # still here? output.
+                print OUT $lead, "\t$_\n"
+            }
         }
     }
 }