Bug 13568 OAI Server doesn't handle properly resumption token

When responding to ListRecords and ListIdentifiers verbs, OAI server doesn't
return proper resumption token. At the end of a result set, OAI server
generates a resumption token even if there isn't anymore records. Consequently,
OAI harverster will send a new request, based on this invalid resumption,
token. OAI Server responds with an empty resultset, which is considered as an
invalid response by most of the harvesters.

TO TEST:

- Find in your DB, a day where a few biblio records have been created. The
  number of created biblios must inferior to OAI-PMH:MaxCount.

- Let say this day is 2014-01-09. Send an OAI-PMH request to Koha OAI Server:

  /cgi-bin/koha/oai.pl?verb=ListRecords&metadataPrefix=marcxml&from=2014-01-09&until=2014-01-09

- At the end of the result, you will see a resumption token which looks like that:

  <resumptionToken cursor="47">marcxml/47/2014-01-09/2014-01-09/</resumptionToken>

  This is wrong. No resumptiion token should be sent since there isn't anymore
  records to harvest.

- Apply the patch.

- Resend the OAI-PMH request. There is no resumption token at the end of the
  result.

- You could test also with ListIdenfiers verb in place of ListRecord.

Signed-off-by: Christophe Brocquet <christophe.brocquet@obspm.fr>

Signed-off-by: Jonathan Druart <jonathan.druart@biblibre.com>
Signed-off-by: Tomas Cohen Arazi <tomascohen@gmail.com>

diff --git a/opac/oai.pl b/opac/oai.pl
index d6ef2e9..137a5f6 100755 (executable)
--- a/opac/oai.pl
+++ b/opac/oai.pl
@@ -71,7 +71,7 @@ my $repository = C4::OAI::Repository->new();
 # - from
 # - until
 # - offset
-# 
+#
 package C4::OAI::ResumptionToken;
 
 use strict;
@@ -314,6 +314,7 @@ sub new {
     if(defined $token->{'set'}) {
         $set = GetOAISetBySpec($token->{'set'});
     }
+    my $max = $repository->{koha_max_count};
     my $sql = "
         SELECT biblioitems.biblionumber, biblioitems.timestamp
         FROM biblioitems
@@ -322,32 +323,35 @@ sub new {
     $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? ";
     $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
     $sql .= "
-        LIMIT $repository->{'koha_max_count'}
-        OFFSET $token->{'offset'}
+        LIMIT " . ($max+1) . "
+        OFFSET $token->{offset}
     ";
     my $sth = $dbh->prepare( $sql );
     my @bind_params = ($token->{'from'}, $token->{'until'});
     push @bind_params, $set->{'id'} if defined $set;
     $sth->execute( @bind_params );
 
-    my $pos = $token->{offset};
+    my $count = 0;
     while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) {
+        $count++;
+        if ( $count > $max ) {
+            $self->resumptionToken(
+                new C4::OAI::ResumptionToken(
+                    metadataPrefix  => $token->{metadata_prefix},
+                    from            => $token->{from},
+                    until           => $token->{until},
+                    offset          => $token->{offset} + $max,
+                    set             => $token->{set}
+                )
+            );
+            last;
+        }
         $timestamp =~ s/ /T/, $timestamp .= 'Z';
         $self->identifier( new HTTP::OAI::Header(
             identifier => $repository->{ koha_identifier} . ':' . $biblionumber,
             datestamp  => $timestamp,
         ) );
-        $pos++;
     }
-    $self->resumptionToken(
-        new C4::OAI::ResumptionToken(
-            metadataPrefix  => $token->{metadata_prefix},
-            from            => $token->{from},
-            until           => $token->{until},
-            offset          => $pos,
-            set             => $token->{set}
-        )
-    ) if ($pos > $token->{offset});
 
     return $self;
 }
@@ -467,6 +471,7 @@ sub new {
     if(defined $token->{'set'}) {
         $set = GetOAISetBySpec($token->{'set'});
     }
+    my $max = $repository->{koha_max_count};
     my $sql = "
         SELECT biblioitems.biblionumber, biblioitems.marcxml, biblioitems.timestamp
         FROM biblioitems
@@ -475,8 +480,8 @@ sub new {
     $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? ";
     $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set;
     $sql .= "
-        LIMIT $repository->{'koha_max_count'}
-        OFFSET $token->{'offset'}
+        LIMIT " . ($max + 1) . "
+        OFFSET $token->{offset}
     ";
 
     my $sth = $dbh->prepare( $sql );
@@ -484,8 +489,21 @@ sub new {
     push @bind_params, $set->{'id'} if defined $set;
     $sth->execute( @bind_params );
 
-    my $pos = $token->{offset};
+    my $count = 0;
     while ( my ($biblionumber, $marcxml, $timestamp) = $sth->fetchrow ) {
+        $count++;
+        if ( $count > $max ) {
+            $self->resumptionToken(
+                new C4::OAI::ResumptionToken(
+                    metadataPrefix  => $token->{metadata_prefix},
+                    from            => $token->{from},
+                    until           => $token->{until},
+                    offset          => $token->{offset} + $max,
+                    set             => $token->{set}
+                )
+            );
+            last;
+        }
         my $oai_sets = GetOAISetsBiblio($biblionumber);
         my @setSpecs;
         foreach (@$oai_sets) {
@@ -496,17 +514,7 @@ sub new {
             identifier      => $repository->{ koha_identifier } . ':' . $biblionumber,
             metadataPrefix  => $token->{metadata_prefix}
         ) );
-        $pos++;
     }
-    $self->resumptionToken(
-        new C4::OAI::ResumptionToken(
-            metadataPrefix  => $token->{metadata_prefix},
-            from            => $token->{from},
-            until           => $token->{until},
-            offset          => $pos,
-            set             => $token->{set}
-        )
-    ) if ($pos > $token->{offset});
 
     return $self;
 }
@@ -639,7 +647,7 @@ C4::OAI::Repository - Handles OAI-PMH requests for a Koha database.
 This object extend HTTP::OAI::Repository object.
 It accepts OAI-PMH HTTP requests and returns result.
 
-This OAI-PMH server can operate in a simple mode and extended one. 
+This OAI-PMH server can operate in a simple mode and extended one.
 
 In simple mode, repository configuration comes entirely from Koha system
 preferences (OAI-PMH:archiveID and OAI-PMH:MaxCount) and the server returns