Merge branch 'develop-787' into develop

* develop-787:
  New scripts
  Add experimental source to person "extended" relation
  Remove unused variable
  Rename function to be consistent
  Cleanup and document
  Functional implementation omn 787
  Add experimental migration to add 'id' to sources_to_sources so that activerecord can have a primary key
  Fix non foreign_link links (i.e. 773) so they are not hardcoded and come from configuration
  Initial 787 implementation, remove uniused code and variables
  Initial test with has_many
  Rename "informazioni di possesso" to "esemplari" for comformity to other languages.
  Update changelog
  Hide tag 787

# Conflicts:
#	app/models/source.rb

CHANGELOG

@@ -1,6 +1,9 @@
 Muscat Changelog
 ================
 
+v 7.0.3
+* Temporarily hide 787 tag
+
 v 7.0.2
 * Fix asset compilation in 7.0.2: Merge pull request #1062 from rism-ch/fix-asset-precompile-7.0.2
 * Fix links in catalog so they go to catalog searches

app/models/person.rb

@@ -38,7 +38,10 @@ def user_name
   has_many :works
   has_many :digital_object_links, :as => :object_link, :dependent => :delete_all
   has_many :digital_objects, through: :digital_object_links, foreign_key: "object_link_id"
-  has_and_belongs_to_many(:referring_sources, class_name: "Source", join_table: "sources_to_people")
+  #has_and_belongs_to_many(:referring_sources, class_name: "Source", join_table: "sources_to_people")
+  has_many :source_person_relations, class_name: "SourcePersonRelation"
+  has_many :referring_sources, through: :source_person_relations, source: :source
+
   has_and_belongs_to_many(:referring_institutions, class_name: "Institution", join_table: "institutions_to_people")
   has_and_belongs_to_many(:referring_publications, class_name: "Publication", join_table: "publications_to_people")
   has_and_belongs_to_many(:referring_holdings, class_name: "Holding", join_table: "holdings_to_people")

app/models/source.rb

@@ -44,7 +44,6 @@ class Source < ApplicationRecord
 
   # include the override for group_values
   require 'solr_search.rb'
-#  include MarcIndex
   include ForeignLinks
   include MarcIndex
   include Template
@@ -55,7 +54,10 @@ class Source < ApplicationRecord
   has_many :digital_object_links, :as => :object_link, :dependent => :delete_all
   has_many :digital_objects, through: :digital_object_links, foreign_key: "object_link_id"
   has_and_belongs_to_many :institutions, join_table: "sources_to_institutions"
-  has_and_belongs_to_many :people, join_table: "sources_to_people"
+  #has_and_belongs_to_many :people, join_table: "sources_to_people"
+  has_many :source_person_relations
+  has_many :people, through: :source_person_relations
+
   has_and_belongs_to_many :standard_titles, join_table: "sources_to_standard_titles"
   has_and_belongs_to_many :standard_terms, join_table: "sources_to_standard_terms"
   has_and_belongs_to_many :publications, join_table: "sources_to_publications"
@@ -67,20 +69,15 @@ class Source < ApplicationRecord
   has_many :folder_items, as: :item, dependent: :destroy
   has_many :folders, through: :folder_items, foreign_key: "item_id"
   belongs_to :user, :foreign_key => "wf_owner"
-
-  # This is the forward link
-  has_and_belongs_to_many(:sources,
-    :class_name => "Source",
-    :foreign_key => "source_a_id",
-    :association_foreign_key => "source_b_id",
-    join_table: "sources_to_sources")
-
-  # This is the backward link
-  has_and_belongs_to_many(:referring_sources,
-    :class_name => "Source",
-    :foreign_key => "source_b_id",
-    :association_foreign_key => "source_a_id",
-    join_table: "sources_to_sources")
+
+  # source-to-source many-to-many relation
+  # We need to switch to has_many to use an intermediate model
+  # This is the forward relationship
+  has_many :source_relations, foreign_key: "source_a_id"
+  has_many :sources, through: :source_relations, source: :source_b
+  # And this is the one coming back, i.e. sources pointing to this one from 775
+  has_many :referring_source_relations, class_name: "SourceRelation", foreign_key: "source_b_id"
+  has_many :referring_sources, through: :referring_source_relations, source: :source_a
 
   composed_of :marc, :class_name => "MarcSource", :mapping => [%w(marc_source to_marc), %w(record_type record_type)]
   alias_attribute :id_for_fulltext, :id

app/models/source_person_relation.rb

@@ -0,0 +1,5 @@
+class SourcePersonRelation < ApplicationRecord
+    self.table_name = "sources_to_people"
+    belongs_to :source
+    belongs_to :person
+end

app/models/source_relation.rb

@@ -0,0 +1,5 @@
+class SourceRelation < ApplicationRecord
+    self.table_name = "sources_to_sources"
+    belongs_to :source_a, class_name: "Source"
+    belongs_to :source_b, class_name: "Source"
+end

config/editor_profiles/default/configurations/SourceFormOptions.yml

@@ -915,13 +915,24 @@
     - - w
       - type: source
         editor_partial: subfield_secondary
-"787": 
+"787":
+  new_window: sources
   layout: 
     fields: 
-    - - g
+    - - i
+      - editor_partial: subfield_select_codes
+        codes_reference: "relator_codes"
     - - n
-    - - s
     - - w
+      - type: source
+        editor_partial: subfield_secondary
+  tag_params:
+      relator_codes: 
+      - ts1
+      - ts2
+      - ts3
+      - ts4
+      - ts5
 "852": 
   new_window: institutions
   layout: 

config/locales/it.yml

@@ -168,8 +168,8 @@ it:
         one: "Gruppo"
         other: "Gruppi"
       holding:
-        one: "Informazione di possesso"
-        other: "Informazioni di possesso"
+        one: "Esemplare"
+        other: "Esemplari"
       institution:
         one: "Istituzione"
         other: "Istituzioni"
@@ -309,7 +309,7 @@ it:
   running: "In esecuzione"
   finished: "Completato"
   lib_siglum: "Sigla della biblioteca"
-  holding_records: "Informazioni di possesso"
+  holding_records: "Esemplari"
   holding_no_siglum: "Esemplare senza sigla di biblioteca"
   new_holding: "Nuovo Esemplare"
   new_holding_page: "Esemplare non salvato"

config/marc/tag_config_source.yml

@@ -617,6 +617,7 @@
       - :occurrences: "?"
   "700":
     :master: "0"
+    :relator_code: "4"
     :indicator: "1#"
     :occurrences: "*"
     :fields:
@@ -698,6 +699,7 @@
     :master: w
     :indicator: "18"
     :occurrences: "?"
+    :foreign_links: false
     :fields:
     - - w
       - :occurrences: "1"
@@ -727,21 +729,19 @@
         :zero_padding: "14"
         :browse_helper: application_helper_link_source_id
   "787":
-    :master: a
-    :indicator: "18"
+    :master: "w"
+    :relator_code: "i"
+    :indicator: "0#"
     :occurrences: "*"
     :fields:
-    - - g
-      - :occurrences: "*"
-        :no_browse: true
+    - - i
+      - :occurrences: "?"
     - - n
-      - :occurrences: "*"
-        :no_browse: true
-    - - s
       - :occurrences: "?"
-        :no_browse: true
     - - w
-      - :occurrences: "*"
+      - :occurrences: "1"
+        :foreign_class: Source
+        :foreign_field: id
         :browse_helper: application_helper_link_source_id
   "852":
     :master: "x"

db/migrate/20210317115753_add_relator_code_to_sources_to_sources.rb

@@ -0,0 +1,6 @@
+class AddRelatorCodeToSourcesToSources < ActiveRecord::Migration[5.2]
+  def change
+    add_column :sources_to_sources, :marc_tag, :string
+    add_column :sources_to_sources, :relator_code, :string
+  end
+end

db/migrate/20210317135955_add_id_to_sources_to_sources.rb

@@ -0,0 +1,15 @@
+class AddIdToSourcesToSources < ActiveRecord::Migration[5.2]
+  def self.up
+    execute("ALTER TABLE `sources_to_sources` ADD `id` INT")
+    execute("ALTER TABLE `sources_to_sources` MODIFY COLUMN `id` INT NOT NULL UNIQUE AUTO_INCREMENT FIRST;")
+    execute("ALTER TABLE `sources_to_sources` ADD PRIMARY KEY (`id`)");
+    execute("ALTER TABLE `sources_to_sources` ADD UNIQUE INDEX `unique_records` (`marc_tag`, `relator_code`, `source_a_id`, `source_b_id`);")
+    execute("UPDATE sources_to_sources SET marc_tag = 775;")
+  end
+
+  def self.down
+    execute("ALTER TABLE `sources_to_sources` DROP INDEX `unique_records`;")
+    execute("ALTER TABLE `sources_to_sources` DROP PRIMARY KEY;")
+    execute("ALTER TABLE `sources_to_sources` DROP `id`;")
+  end
+end

db/migrate/20210324123058_extend_sources_to_people.rb

@@ -0,0 +1,6 @@
+class ExtendSourcesToPeople < ActiveRecord::Migration[5.2]
+  def change
+    add_column :sources_to_people, :marc_tag, :string
+    add_column :sources_to_people, :relator_code, :string
+  end
+end

db/migrate/20210324123135_add_id_to_sources_to_people.rb

@@ -0,0 +1,14 @@
+class AddIdToSourcesToPeople < ActiveRecord::Migration[5.2]
+  def self.up
+    execute("ALTER TABLE `sources_to_people` ADD `id` INT")
+    execute("ALTER TABLE `sources_to_people` MODIFY COLUMN `id` INT NOT NULL UNIQUE AUTO_INCREMENT FIRST;")
+    execute("ALTER TABLE `sources_to_people` ADD PRIMARY KEY (`id`)");
+    execute("ALTER TABLE `sources_to_people` ADD UNIQUE INDEX `unique_records` (`marc_tag`, `relator_code`, `source_id`, `person_id`);")
+  end
+
+  def self.down
+    execute("ALTER TABLE `sources_to_people` DROP INDEX `unique_records`;")
+    execute("ALTER TABLE `sources_to_people` DROP PRIMARY KEY;")
+    execute("ALTER TABLE `sources_to_people` DROP `id`;")
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_11_24_090059) do
+ActiveRecord::Schema.define(version: 2021_03_24_123135) do
 
   create_table "active_admin_comments", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
     t.string "namespace"
@@ -446,9 +446,13 @@
     t.index ["source_id"], name: "index_sources_to_liturgical_feasts_on_source_id"
   end
 
-  create_table "sources_to_people", id: false, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+  create_table "sources_to_people", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
     t.integer "person_id"
     t.integer "source_id"
+    t.string "marc_tag"
+    t.string "relator_code"
+    t.index ["id"], name: "id", unique: true
+    t.index ["marc_tag", "relator_code", "source_id", "person_id"], name: "unique_records", unique: true
     t.index ["person_id"], name: "index_sources_to_people_on_person_id"
     t.index ["source_id"], name: "index_sources_to_people_on_source_id"
   end
@@ -467,9 +471,13 @@
     t.index ["source_id"], name: "index_sources_to_publications_on_source_id"
   end
 
-  create_table "sources_to_sources", id: false, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+  create_table "sources_to_sources", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
     t.integer "source_a_id"
     t.integer "source_b_id"
+    t.string "marc_tag"
+    t.string "relator_code"
+    t.index ["id"], name: "id", unique: true
+    t.index ["marc_tag", "relator_code", "source_a_id", "source_b_id"], name: "unique_records", unique: true
     t.index ["source_a_id"], name: "index_sources_to_sources_on_source_a_id"
     t.index ["source_b_id"], name: "index_sources_to_sources_on_source_b_id"
   end

housekeeping/correct/52_tag_usage.rb

@@ -0,0 +1,35 @@
+tag_count = {}
+pb = ProgressBar.new(Source.count)
+
+Source.find_in_batches do |batch|
+
+    batch.each do |src|
+        s = Source.find(src.id)
+        pb.increment!
+
+        s.marc.load_source false
+        marc = s.marc
+
+        marc.all_tags.each do |tag|
+
+            if !tag_count.include?(tag.tag)
+                tag_count[tag.tag] = {}
+                tag_count[tag.tag][:count] = 0
+                tag_count[tag.tag][:subtags] = {}
+            end
+
+            tag_count[tag.tag][:count] += 1
+
+            tag.children.each do |subtag|
+                if !tag_count[tag.tag][:subtags].include?(subtag.tag)
+                    tag_count[tag.tag][:subtags][subtag.tag] = {}
+                    tag_count[tag.tag][:subtags][subtag.tag][:count] = 0
+                    tag_count[tag.tag][:subtags][subtag.tag][:first] = src.id
+                end
+                tag_count[tag.tag][:subtags][subtag.tag][:count] += 1
+            end
+        end
+    end
+end
+
+puts tag_count.to_yaml

housekeeping/correct/53_print_tags_as_csv.rb

@@ -0,0 +1,43 @@
+def tag2str
+end
+
+headers = %w(
+    001
+    005
+    240a
+    245a
+    260a
+    260b
+    260c
+    510a
+    510c
+    650a
+    852a
+    588a
+)
+
+CSV.open("referred_sources.csv", "w") do |csv|
+    csv << headers
+
+    Source.joins(:referring_sources).distinct.each do |s|
+        line = []
+
+        line << s.id.to_s
+        line << s.marc.first_occurance("005").content rescue line << ""
+
+        line << s.marc.all_values_for_tags_with_subtag("240","a").join("; ")
+        line << s.marc.all_values_for_tags_with_subtag("245","a").join("; ")
+        line << s.marc.all_values_for_tags_with_subtag("260","a").join("; ")
+        line << s.marc.all_values_for_tags_with_subtag("260","b").join("; ")
+        line << s.marc.all_values_for_tags_with_subtag("260","c").join("; ")
+        line << s.marc.all_values_for_tags_with_subtag("510","a").join("; ")
+        line << s.marc.all_values_for_tags_with_subtag("510","c").join("; ")
+        line << s.marc.all_values_for_tags_with_subtag("650","a").join("; ")
+        line << s.marc.all_values_for_tags_with_subtag("852","a").join("; ")
+        line << s.marc.all_values_for_tags_with_subtag("588","a").join("; ")
+
+        #ap line
+        csv << line
+
+    end
+end