Bug 3206: OAI repository deleted records support.

This patch allows Koha OAI repository to support deleted records.

The OAI-PMH:DeletedRecord syspref is introduced and can be set to:

- persistent (in case Koha's deletedbiblio table will never be emptied
  or truncated)
- transient (in case Koha's deletedbiblio table might be emptied or
  truncated at some point)

Test plan:

- After applying the patch, test that:
   - Deleted records appear in ListRecords and ListIdentifiers requests.
   - Filter parameters (from, until, set and resumptionToken) still work
     and are applied to ListRecords and ListIdentifiers requests.
   - Identify request shows if the repository is considered persistent
     or transient, according to the OAI-PMH:DeletedRecord syspref.
   - Deleted records that used to belong to a set are still displayed in
     those sets and marked as deleted.
   - GetRecord requests work on deleted records, which are marked as deleted.

Requests examples:
/cgi-bin/koha/oai.pl?verb=ListRecords&metadataPrefix=oai_dc
/cgi-bin/koha/oai.pl?verb=ListRecords&metadataPrefix=oai_dc&from=2015-02-20T11:08:33Z
/cgi-bin/koha/oai.pl?verb=ListRecords&metadataPrefix=oai_dc&set=new_specSet1
/cgi-bin/koha/oai.pl?verb=GetRecord&identifier=KOHA-OAI-TEST:2&metadataPrefix=oai_dc
/cgi-bin/koha/oai.pl?verb=Identify

Signed-off-by: Frederic Demians <f.demians@tamil.fr>
  It works in all situations described in the test plan. Great addition.
  Thanks.

Signed-off-by: Jonathan Druart <jonathan.druart@koha-community.org>

Signed-off-by: Jonathan Druart <jonathan.druart@koha-community.org>
Signed-off-by: Tomas Cohen Arazi <tomascohen@theke.io>

diff --git a/installer/data/mysql/atomicupdate/bug_XXXX-add_OAI-PMH_DeletedRecord_syspref.sql b/installer/data/mysql/atomicupdate/bug_XXXX-add_OAI-PMH_DeletedRecord_syspref.sql
new file mode 100644 (file)
index 0000000..233bcbe
--- /dev/null
+++ b/installer/data/mysql/atomicupdate/bug_XXXX-add_OAI-PMH_DeletedRecord_syspref.sql
@@ -0,0 +1,3 @@
+INSERT IGNORE INTO systempreferences (variable,value,explanation,options,type)
+VALUES ('OAI-PMH:DeletedRecord','persistent','Koha\'s deletedbiblio table will never be deleted (persistent) or might be deleted (transient)','transient|persistent','Choice');
+ALTER TABLE oai_sets_biblios DROP FOREIGN KEY oai_sets_biblios_ibfk_1;

diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql
index 66895d1..df123ee 100644 (file)
--- a/installer/data/mysql/kohastructure.sql
+++ b/installer/data/mysql/kohastructure.sql
@@ -1610,7 +1610,6 @@ CREATE TABLE `oai_sets_biblios` (
   `biblionumber` int(11) NOT NULL,
   `set_id` int(11) NOT NULL,
   PRIMARY KEY (`biblionumber`, `set_id`),
-  CONSTRAINT `oai_sets_biblios_ibfk_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE CASCADE,
   CONSTRAINT `oai_sets_biblios_ibfk_2` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
 

diff --git a/installer/data/mysql/sysprefs.sql b/installer/data/mysql/sysprefs.sql
index 84ba363..f7ea475 100644 (file)
--- a/installer/data/mysql/sysprefs.sql
+++ b/installer/data/mysql/sysprefs.sql
@@ -480,5 +480,6 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `
 ('XSLTDetailsDisplay','default','','Enable XSL stylesheet control over details page display on intranet','Free'),
 ('XSLTResultsDisplay','default','','Enable XSL stylesheet control over results page display on intranet','Free'),
 ('z3950AuthorAuthFields','701,702,700',NULL,'Define the MARC biblio fields for Personal Name Authorities to fill biblio.author','free'),
-('z3950NormalizeAuthor','0','','If ON, Personal Name Authorities will replace authors in biblio.author','YesNo')
+('z3950NormalizeAuthor','0','','If ON, Personal Name Authorities will replace authors in biblio.author','YesNo'),
+('OAI-PMH:DeletedRecord','persistent','Koha\'s deletedbiblio table will never be deleted (persistent) or might be deleted (transient)','transient|persistent','Choice')
 ;

diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref
index 075ffb7..518dfbd 100644 (file)
--- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref
+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref
@@ -27,6 +27,13 @@ Web services:
                   yes: Enable
                   no: Disable
             - automatic update of OAI-PMH sets when a bibliographic record is created or updated
+        -
+            - Koha's deletedbiblio table
+            - pref: "OAI-PMH:DeletedRecord"
+              choices:
+                  persistent: will never be emptied or truncated (persistent)
+                  transient: might be emptied or truncated at some point (transient)
+            - "."
     ILS-DI:
         -
             - pref: ILS-DI

diff --git a/opac/oai.pl b/opac/oai.pl
index c80ebfa..82af58c 100755 (executable)
--- a/opac/oai.pl
+++ b/opac/oai.pl
@@ -152,7 +152,7 @@ sub new {
         MaxCount            => C4::Context->preference("OAI-PMH:MaxCount"),
         granularity         => 'YYYY-MM-DD',
         earliestDatestamp   => '0001-01-01',
-        deletedRecord       => 'no',
+        deletedRecord       => C4::Context->preference("OAI-PMH:DeletedRecord") || 'no',
     );
 
     # FIXME - alas, the description element is not so simple; to validate
@@ -252,6 +252,36 @@ sub new {
 
 # __END__ C4::OAI::Record
 
+package C4::OAI::DeletedRecord;
+
+use strict;
+use warnings;
+use HTTP::OAI;
+use HTTP::OAI::Metadata::OAI_DC;
+
+use base ("HTTP::OAI::Record");
+
+sub new {
+    my ($class, $timestamp, $setSpecs, %args) = @_;
+
+    my $self = $class->SUPER::new(%args);
+
+    $timestamp =~ s/ /T/, $timestamp .= 'Z';
+    $self->header( new HTTP::OAI::Header(
+        status      => 'deleted',
+        identifier  => $args{identifier},
+        datestamp   => $timestamp,
+    ) );
+
+    foreach my $setSpec (@$setSpecs) {
+        $self->header->setSpec($setSpec);
+    }
+
+    return $self;
+}
+
+# __END__ C4::OAI::DeletedRecord
+
 
 
 package C4::OAI::GetRecord;
@@ -278,7 +308,16 @@ sub new {
     my ($biblionumber) = $args{identifier} =~ /^$prefix(.*)/;
     $sth->execute( $biblionumber );
     my ($marcxml, $timestamp);
+    my $deleted = 0;
     unless ( ($marcxml, $timestamp) = $sth->fetchrow ) {
+      $sth = $dbh->prepare("
+        SELECT biblionumber, timestamp
+        FROM deletedbiblio
+        WHERE biblionumber=? " );
+      $sth->execute( $biblionumber );
+
+      unless ( ($marcxml, $timestamp) = $sth->fetchrow ) {
+
         return HTTP::OAI::Response->new(
             requestURL  => $repository->self_url(),
             errors      => [ new HTTP::OAI::Error(
@@ -286,8 +325,10 @@ sub new {
                 message => "There is no biblio record with this identifier",
                 ) ] ,
         );
+      } else {
+        $deleted = 1;
+      }
     }
-
     my $oai_sets = GetOAISetsBiblio($biblionumber);
     my @setSpecs;
     foreach (@$oai_sets) {
@@ -295,9 +336,10 @@ sub new {
     }
 
     #$self->header( HTTP::OAI::Header->new( identifier  => $args{identifier} ) );
-    $self->record( C4::OAI::Record->new(
+    ($deleted == 1) ? $self->record( C4::OAI::DeletedRecord->new(
+        $timestamp, \@setSpecs, %args ) )
+        : $self->record( C4::OAI::Record->new(
         $repository, $marcxml, $timestamp, \@setSpecs, %args ) );
-
     return $self;
 }
 
@@ -328,19 +370,27 @@ sub new {
     }
     my $max = $repository->{koha_max_count};
     my $sql = "
-        SELECT biblioitems.biblionumber, biblioitems.timestamp
+        (SELECT biblioitems.biblionumber, timestamp
         FROM biblioitems
     ";
     $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set;
     $sql .= " WHERE timestamp >= ? AND timestamp <= ? ";
     $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
-    $sql .= "
+    $sql .= ") UNION
+        (SELECT deletedbiblio.biblionumber, timestamp FROM deletedbiblio";
+    $sql .= " JOIN oai_sets_biblios ON deletedbiblio.biblionumber = oai_sets_biblios.biblionumber " if defined $set;
+    $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? ";
+    $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
+
+    $sql .= ") ORDER BY biblionumber
         LIMIT " . ($max+1) . "
         OFFSET $token->{offset}
     ";
     my $sth = $dbh->prepare( $sql );
     my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'});
     push @bind_params, $set->{'id'} if defined $set;
+    push @bind_params, ($token->{'from'}, $token->{'until'});
+    push @bind_params, $set->{'id'} if defined $set;
     $sth->execute( @bind_params );
 
     my $count = 0;
@@ -485,20 +535,27 @@ sub new {
     }
     my $max = $repository->{koha_max_count};
     my $sql = "
-        SELECT biblioitems.biblionumber, biblioitems.marcxml, biblioitems.timestamp
+        (SELECT biblioitems.biblionumber, marcxml, timestamp
         FROM biblioitems
     ";
     $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set;
     $sql .= " WHERE timestamp >= ? AND timestamp <= ? ";
     $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
-    $sql .= "
+    $sql .= ") UNION
+        (SELECT deletedbiblio.biblionumber, null as marcxml, timestamp FROM deletedbiblio";
+    $sql .= " JOIN oai_sets_biblios ON deletedbiblio.biblionumber = oai_sets_biblios.biblionumber " if defined $set;
+    $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? ";
+    $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
+
+    $sql .= ") ORDER BY biblionumber
         LIMIT " . ($max + 1) . "
         OFFSET $token->{offset}
     ";
-
     my $sth = $dbh->prepare( $sql );
     my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'});
     push @bind_params, $set->{'id'} if defined $set;
+    push @bind_params, ($token->{'from'}, $token->{'until'});
+    push @bind_params, $set->{'id'} if defined $set;
     $sth->execute( @bind_params );
 
     my $count = 0;
@@ -521,11 +578,16 @@ sub new {
         foreach (@$oai_sets) {
             push @setSpecs, $_->{spec};
         }
-        $self->record( C4::OAI::Record->new(
-            $repository, $marcxml, $timestamp, \@setSpecs,
-            identifier      => $repository->{ koha_identifier } . ':' . $biblionumber,
-            metadataPrefix  => $token->{metadata_prefix}
-        ) );
+        if ($marcxml) {
+          $self->record( C4::OAI::Record->new(
+              $repository, $marcxml, $timestamp, \@setSpecs,
+              identifier      => $repository->{ koha_identifier } . ':' . $biblionumber,
+              metadataPrefix  => $token->{metadata_prefix}
+          ) );
+        } else {
+          $self->record( C4::OAI::DeletedRecord->new(
+          $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber ) );
+        }
     }
 
     return $self;