From 264c2f644554322efdcc801cc99a2edafe51e2a9 Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Mon, 1 May 2023 11:51:42 -0400 Subject: [PATCH] LP#1979071: (follow-up) fix various DB schema and update issues * Incorporate the delayed reification option for search dictionary updates derived from authority records (see bug 1997485) * Fixed an issue that prevented propagation of changes to main headings * Fixed an issue introduced in bug 1977485 that prevented authority headings from being deleted * Remove now-disused authority.indexing_ingest_or_delete() * Remove an older version of authority.propagate_changes() from the base schema; this would be an issue in new databases installed from scratch. Signed-off-by: Galen Charlton --- Open-ILS/src/sql/Pg/030.schema.metabib.sql | 5 + Open-ILS/src/sql/Pg/090.schema.action.sql | 7 +- Open-ILS/src/sql/Pg/300.schema.staged_search.sql | 9 ++- Open-ILS/src/sql/Pg/999.functions.global.sql | 127 -------------------- .../sql/Pg/upgrade/XXXX.schema.queued_ingest.sql | 68 ++++++++++- 5 files changed, 82 insertions(+), 134 deletions(-) diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql index 0176bae..a3f9ec3 100644 --- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql +++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql @@ -2086,6 +2086,11 @@ BEGIN END IF; END IF; + PERFORM * FROM config.internal_flag WHERE name = 'ingest.disable_symspell_reification' AND enabled; + IF NOT FOUND THEN + PERFORM search.symspell_dictionary_reify(); + END IF; + RETURN TRUE; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS diag_detail = PG_EXCEPTION_DETAIL, diff --git a/Open-ILS/src/sql/Pg/090.schema.action.sql b/Open-ILS/src/sql/Pg/090.schema.action.sql index 1c5572a..69d6b03 100644 --- a/Open-ILS/src/sql/Pg/090.schema.action.sql +++ b/Open-ILS/src/sql/Pg/090.schema.action.sql @@ -1925,6 +1925,7 @@ CREATE OR REPLACE FUNCTION action.process_ingest_queue_entry (qeid BIGINT) RETUR DECLARE ingest_success BOOL := NULL; qe action.ingest_queue_entry%ROWTYPE; + aid authority.record_entry.id%TYPE; BEGIN SELECT * INTO qe FROM action.ingest_queue_entry WHERE id = qeid; @@ -1941,7 +1942,8 @@ BEGIN ELSE IF qe.record_type = 'biblio' THEN IF qe.action = 'propagate' THEN - SELECT authority.apply_propagate_changes(qe.state_data::BIGINT, qe.record) INTO ingest_success; + SELECT authority.apply_propagate_changes(qe.state_data::BIGINT, qe.record) INTO aid; + SELECT aid = qe.state_data::BIGINT INTO ingest_success; ELSE SELECT metabib.indexing_update(r.*, qe.action = 'insert', qe.state_data) INTO ingest_success FROM biblio.record_entry r WHERE r.id = qe.record; END IF; @@ -1966,7 +1968,6 @@ BEGIN END; $func$ LANGUAGE PLPGSQL; - CREATE OR REPLACE FUNCTION action.complete_duplicated_entries () RETURNS TRIGGER AS $F$ BEGIN IF NEW.ingest_time IS NOT NULL THEN @@ -2018,7 +2019,7 @@ BEGIN IF FOUND THEN -- XXX enqueue special 'propagate' bib action - SELECT action.enqueue_ingest_entry( bid, 'biblio', NOW(), 'propagate', aid::TEXT) INTO queuing_success; + SELECT action.enqueue_ingest_entry( bid, 'biblio', NOW(), NULL, 'propagate', aid::TEXT) INTO queuing_success; IF queuing_success THEN RETURN aid; diff --git a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql index fd11b3a..c9656cf 100644 --- a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql +++ b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql @@ -2261,13 +2261,20 @@ DECLARE search_class TEXT; new_value TEXT := NULL; old_value TEXT := NULL; + _atag INTEGER; BEGIN IF TG_TABLE_SCHEMA = 'authority' THEN + IF TG_OP IN ('INSERT', 'UPDATE') THEN + _atag = NEW.atag; + ELSE + _atag = OLD.atag; + END IF; + SELECT m.field_class INTO search_class FROM authority.control_set_auth_field_metabib_field_map_refs a JOIN config.metabib_field m ON (a.metabib_field=m.id) - WHERE a.authority_field = NEW.atag; + WHERE a.authority_field = _atag; IF NOT FOUND THEN RETURN NULL; diff --git a/Open-ILS/src/sql/Pg/999.functions.global.sql b/Open-ILS/src/sql/Pg/999.functions.global.sql index d115364..aebdc72 100644 --- a/Open-ILS/src/sql/Pg/999.functions.global.sql +++ b/Open-ILS/src/sql/Pg/999.functions.global.sql @@ -1333,45 +1333,6 @@ END; $func$ LANGUAGE plpgsql; -- Authority ingest routines -CREATE OR REPLACE FUNCTION authority.propagate_changes - (aid BIGINT, bid BIGINT) RETURNS BIGINT AS $func$ -DECLARE - bib_rec biblio.record_entry%ROWTYPE; - new_marc TEXT; -BEGIN - - SELECT INTO bib_rec * FROM biblio.record_entry WHERE id = bid; - - new_marc := vandelay.merge_record_xml( - bib_rec.marc, authority.generate_overlay_template(aid)); - - IF new_marc = bib_rec.marc THEN - -- Authority record change had no impact on this bib record. - -- Nothing left to do. - RETURN aid; - END IF; - - PERFORM 1 FROM config.global_flag - WHERE name = 'ingest.disable_authority_auto_update_bib_meta' - AND enabled; - - IF NOT FOUND THEN - -- update the bib record editor and edit_date - bib_rec.editor := ( - SELECT editor FROM authority.record_entry WHERE id = aid); - bib_rec.edit_date = NOW(); - END IF; - - UPDATE biblio.record_entry SET - marc = new_marc, - editor = bib_rec.editor, - edit_date = bib_rec.edit_date - WHERE id = bid; - - RETURN aid; - -END; -$func$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION authority.propagate_changes (aid BIGINT) RETURNS SETOF BIGINT AS $func$ SELECT authority.propagate_changes( authority, bib ) FROM authority.bib_linking WHERE authority = $1; @@ -1478,94 +1439,6 @@ BEGIN END; $func$ LANGUAGE PLPGSQL; --- AFTER UPDATE OR INSERT trigger for authority.record_entry -CREATE OR REPLACE FUNCTION authority.indexing_ingest_or_delete () RETURNS TRIGGER AS $func$ -DECLARE - ashs authority.simple_heading%ROWTYPE; - mbe_row metabib.browse_entry%ROWTYPE; - mbe_id BIGINT; - ash_id BIGINT; -BEGIN - - IF NEW.deleted IS TRUE THEN -- If this authority is deleted - DELETE FROM authority.bib_linking WHERE authority = NEW.id; -- Avoid updating fields in bibs that are no longer visible - DELETE FROM authority.full_rec WHERE record = NEW.id; -- Avoid validating fields against deleted authority records - DELETE FROM authority.simple_heading WHERE record = NEW.id; - -- Should remove matching $0 from controlled fields at the same time? - - -- XXX What do we about the actual linking subfields present in - -- authority records that target this one when this happens? - DELETE FROM authority.authority_linking - WHERE source = NEW.id OR target = NEW.id; - - RETURN NEW; -- and we're done - END IF; - - IF TG_OP = 'UPDATE' THEN -- re-ingest? - PERFORM * FROM config.internal_flag WHERE name = 'ingest.reingest.force_on_same_marc' AND enabled; - - IF NOT FOUND AND OLD.marc = NEW.marc THEN -- don't do anything if the MARC didn't change - RETURN NEW; - END IF; - - -- Unless there's a setting stopping us, propagate these updates to any linked bib records when the heading changes - PERFORM * FROM config.internal_flag WHERE name = 'ingest.disable_authority_auto_update' AND enabled; - - IF NOT FOUND AND NEW.heading <> OLD.heading THEN - PERFORM authority.propagate_changes(NEW.id); - END IF; - - DELETE FROM authority.simple_heading WHERE record = NEW.id; - DELETE FROM authority.authority_linking WHERE source = NEW.id; - END IF; - - INSERT INTO authority.authority_linking (source, target, field) - SELECT source, target, field FROM authority.calculate_authority_linking( - NEW.id, NEW.control_set, NEW.marc::XML - ); - - FOR ashs IN SELECT * FROM authority.simple_heading_set(NEW.marc) LOOP - - INSERT INTO authority.simple_heading (record,atag,value,sort_value,thesaurus) - VALUES (ashs.record, ashs.atag, ashs.value, ashs.sort_value, ashs.thesaurus); - ash_id := CURRVAL('authority.simple_heading_id_seq'::REGCLASS); - - SELECT INTO mbe_row * FROM metabib.browse_entry - WHERE value = ashs.value AND sort_value = ashs.sort_value; - - IF FOUND THEN - mbe_id := mbe_row.id; - ELSE - INSERT INTO metabib.browse_entry - ( value, sort_value ) VALUES - ( ashs.value, ashs.sort_value ); - - mbe_id := CURRVAL('metabib.browse_entry_id_seq'::REGCLASS); - END IF; - - INSERT INTO metabib.browse_entry_simple_heading_map (entry,simple_heading) VALUES (mbe_id,ash_id); - - END LOOP; - - -- Flatten and insert the afr data - PERFORM * FROM config.internal_flag WHERE name = 'ingest.disable_authority_full_rec' AND enabled; - IF NOT FOUND THEN - PERFORM authority.reingest_authority_full_rec(NEW.id); - PERFORM * FROM config.internal_flag WHERE name = 'ingest.disable_authority_rec_descriptor' AND enabled; - IF NOT FOUND THEN - PERFORM authority.reingest_authority_rec_descriptor(NEW.id); - END IF; - END IF; - - PERFORM * FROM config.internal_flag WHERE name = 'ingest.disable_symspell_reification' AND enabled; - IF NOT FOUND THEN - PERFORM search.symspell_dictionary_reify(); - END IF; - - RETURN NEW; -END; -$func$ LANGUAGE PLPGSQL; - -- Ingest triggers CREATE TRIGGER fingerprint_tgr BEFORE INSERT OR UPDATE ON biblio.record_entry FOR EACH ROW EXECUTE PROCEDURE biblio.fingerprint_trigger ('eng','BKS'); CREATE TRIGGER bbb_simple_rec_trigger AFTER INSERT OR UPDATE OR DELETE ON biblio.record_entry FOR EACH ROW EXECUTE PROCEDURE reporter.simple_rec_trigger (); diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.queued_ingest.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.queued_ingest.sql index fe8974d..990247f 100644 --- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.queued_ingest.sql +++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.queued_ingest.sql @@ -103,6 +103,59 @@ INSERT INTO config.global_flag (name, enabled, label) VALUES ( UPDATE config.global_flag SET value = '20' WHERE name = 'ingest.queued.max_threads'; +CREATE OR REPLACE FUNCTION search.symspell_maintain_entries () RETURNS TRIGGER AS $f$ +DECLARE + search_class TEXT; + new_value TEXT := NULL; + old_value TEXT := NULL; + _atag INTEGER; +BEGIN + + IF TG_TABLE_SCHEMA = 'authority' THEN + IF TG_OP IN ('INSERT', 'UPDATE') THEN + _atag = NEW.atag; + ELSE + _atag = OLD.atag; + END IF; + + SELECT m.field_class INTO search_class + FROM authority.control_set_auth_field_metabib_field_map_refs a + JOIN config.metabib_field m ON (a.metabib_field=m.id) + WHERE a.authority_field = _atag; + + IF NOT FOUND THEN + RETURN NULL; + END IF; + ELSE + search_class := COALESCE(TG_ARGV[0], SPLIT_PART(TG_TABLE_NAME,'_',1)); + END IF; + + IF TG_OP IN ('INSERT', 'UPDATE') THEN + new_value := NEW.value; + END IF; + + IF TG_OP IN ('DELETE', 'UPDATE') THEN + old_value := OLD.value; + END IF; + + IF new_value = old_value THEN + -- same, move along + ELSE + INSERT INTO search.symspell_dictionary_updates + SELECT txid_current(), * + FROM search.symspell_build_entries( + new_value, + search_class, + old_value + ); + END IF; + + -- PERFORM * FROM search.symspell_build_and_merge_entries(new_value, search_class, old_value); + + RETURN NULL; -- always fired AFTER +END; +$f$ LANGUAGE PLPGSQL; + CREATE TABLE action.ingest_queue ( id SERIAL PRIMARY KEY, created TIMESTAMPTZ NOT NULL DEFAULT NOW(), @@ -231,6 +284,7 @@ CREATE OR REPLACE FUNCTION action.process_ingest_queue_entry (qeid BIGINT) RETUR DECLARE ingest_success BOOL := NULL; qe action.ingest_queue_entry%ROWTYPE; + aid authority.record_entry.id%TYPE; BEGIN SELECT * INTO qe FROM action.ingest_queue_entry WHERE id = qeid; @@ -247,7 +301,8 @@ BEGIN ELSE IF qe.record_type = 'biblio' THEN IF qe.action = 'propagate' THEN - SELECT authority.apply_propagate_changes(qe.state_data::BIGINT, qe.record) INTO ingest_success; + SELECT authority.apply_propagate_changes(qe.state_data::BIGINT, qe.record) INTO aid; + SELECT aid = qe.state_data::BIGINT INTO ingest_success; ELSE SELECT metabib.indexing_update(r.*, qe.action = 'insert', qe.state_data) INTO ingest_success FROM biblio.record_entry r WHERE r.id = qe.record; END IF; @@ -272,7 +327,6 @@ BEGIN END; $func$ LANGUAGE PLPGSQL; - CREATE OR REPLACE FUNCTION action.complete_duplicated_entries () RETURNS TRIGGER AS $F$ BEGIN IF NEW.ingest_time IS NOT NULL THEN @@ -520,6 +574,11 @@ BEGIN END IF; END IF; + PERFORM * FROM config.internal_flag WHERE name = 'ingest.disable_symspell_reification' AND enabled; + IF NOT FOUND THEN + PERFORM search.symspell_dictionary_reify(); + END IF; + RETURN TRUE; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS diag_detail = PG_EXCEPTION_DETAIL, @@ -887,7 +946,7 @@ BEGIN IF FOUND THEN -- XXX enqueue special 'propagate' bib action - SELECT action.enqueue_ingest_entry( bid, 'biblio', NOW(), 'propagate', aid::TEXT) INTO queuing_success; + SELECT action.enqueue_ingest_entry( bid, 'biblio', NOW(), NULL, 'propagate', aid::TEXT) INTO queuing_success; IF queuing_success THEN RETURN aid; @@ -1080,5 +1139,8 @@ BEGIN END; $func$ LANGUAGE PLPGSQL; +-- get rid of old version +DROP FUNCTION authority.indexing_ingest_or_delete; + COMMIT; -- 1.7.2.5