Merge remote-tracking branch 'IQSS/develop' into IQSS/11126-Payara6.2…

…024.12

conf/solr/schema.xml

@@ -254,6 +254,21 @@
         WARNING: Do not remove the following include guards if you intend to use the neat helper scripts we provide.
     -->
     <!-- SCHEMA-FIELDS::BEGIN -->
+    <field name="3d3DTechnique" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="3dAltText" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dEquipment" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dExportedFileFormat" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="3dExportedFilePolygonCount" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="3dHandling" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dHeight" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dLength" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dLightingSetup" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="3dMasterFilePolygonCount" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dMaterialComposition" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="3dObjectDimensions" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dUnit" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="3dWeight" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="3dWidth" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="accessToSources" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="actionsToMinimizeLoss" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="alternativeTitle" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -496,6 +511,21 @@
         WARNING: Do not remove the following include guards if you intend to use the neat helper scripts we provide.
     -->
     <!-- SCHEMA-COPY-FIELDS::BEGIN -->
+    <copyField source="3d3DTechnique" dest="_text_" maxChars="3000"/>
+    <copyField source="3dAltText" dest="_text_" maxChars="3000"/>
+    <copyField source="3dEquipment" dest="_text_" maxChars="3000"/>
+    <copyField source="3dExportedFileFormat" dest="_text_" maxChars="3000"/>
+    <copyField source="3dExportedFilePolygonCount" dest="_text_" maxChars="3000"/>
+    <copyField source="3dHandling" dest="_text_" maxChars="3000"/>
+    <copyField source="3dHeight" dest="_text_" maxChars="3000"/>
+    <copyField source="3dLength" dest="_text_" maxChars="3000"/>
+    <copyField source="3dLightingSetup" dest="_text_" maxChars="3000"/>
+    <copyField source="3dMasterFilePolygonCount" dest="_text_" maxChars="3000"/>
+    <copyField source="3dMaterialComposition" dest="_text_" maxChars="3000"/>
+    <copyField source="3dObjectDimensions" dest="_text_" maxChars="3000"/>
+    <copyField source="3dUnit" dest="_text_" maxChars="3000"/>
+    <copyField source="3dWeight" dest="_text_" maxChars="3000"/>
+    <copyField source="3dWidth" dest="_text_" maxChars="3000"/>
     <copyField source="accessToSources" dest="_text_" maxChars="3000"/>
     <copyField source="actionsToMinimizeLoss" dest="_text_" maxChars="3000"/>
     <copyField source="alternativeTitle" dest="_text_" maxChars="3000"/>

doc/release-notes/10108-stata-version-detected-in-direct-upload.md

@@ -0,0 +1 @@
+The version of Stata files is now detected during S3 direct upload (as it was for normal uploads), allowing ingest of Stata 14 and 15 files that have been uploaded directly. See [the guides](https://dataverse-guide--11054.org.readthedocs.build/en/11054/developers/big-data-support.html#features-that-are-disabled-if-s3-direct-upload-is-enabled), #10108, and #11054.

doc/release-notes/10320-cookie-consent.md

@@ -0,0 +1,3 @@
+## Cookie Consent Popup (GDPR)
+
+For compliance with GDPR and other privacy regulations, advice on adding a cookie consent popup has been added to the guides. See the new [cookie consent](https://dataverse-guide--10320.org.readthedocs.build/en/10320/installation/config.html#adding-cookie-consent-for-gdpr-etc) section and #10320.

doc/release-notes/10519-dataset-types.md

@@ -0,0 +1,12 @@
+## Dataset Types can be linked to Metadata Blocks
+
+Metadata blocks (e.g. "CodeMeta") can now be linked to dataset types (e.g. "software") using new superuser APIs.
+
+This will have the following effects for the APIs used by the new Dataverse UI ( https://github.com/IQSS/dataverse-frontend ):
+
+- The list of fields shown when creating a dataset will include fields marked as "displayoncreate" (in the tsv/database) for metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API.
+- The metadata blocks shown when editing a dataset will include metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API.
+
+Mostly in order to write automated tests for the above, a [displayOnCreate](https://dataverse-guide--11001.org.readthedocs.build/en/11001/api/native-api.html#set-displayoncreate-for-a-dataset-field) API endpoint has been added.
+
+For more information, see the guides ([overview](https://dataverse-guide--11001.org.readthedocs.build/en/11001/user/dataset-management.html#dataset-types), [new APIs](https://dataverse-guide--11001.org.readthedocs.build/en/11001/api/native-api.html#link-dataset-type-with-metadata-blocks)), #10519 and #11001.

doc/release-notes/10542-signposting.md

@@ -0,0 +1,11 @@
+# Signposting Output Now Contains Links to All Dataset Metadata Export Formats
+
+When Signposting was added in Dataverse 5.14 (#8981), it only provided links for the `schema.org` metadata export format.
+
+The output of HEAD, GET, and the Signposting "linkset" API have all been updated to include links to all available dataset metadata export formats (including any external exporters, such as Croissant, that have been enabled).
+
+This provides a lightweight machine-readable way to first retrieve a list of links (via a HTTP HEAD request, for example) to each available metadata export format and then follow up with a request for the export format of interest.
+
+In addition, the content type for the `schema.org` dataset metadata export format has been corrected. It was `application/json` and now it is `application/ld+json`.
+
+See also [the docs](https://preview.guides.gdcc.io/en/develop/api/native-api.html#retrieve-signposting-information) and #10542.

doc/release-notes/10790-fix pid handling in exporters and citations.md

@@ -0,0 +1,16 @@
+### Improvements to PID formatting in exports and citations
+
+Multiple small issues with the formatting of PIDs in the
+DDI exporters, and EndNote and BibTeX citation formats have
+been addressed. These should improve the ability to import
+Dataverse citations into reference managers and fix potential
+issues harvesting datasets using PermaLinks.
+
+Backward Incompatibility
+
+Changes to PID formatting occur in the DDI/DDI Html export formats
+and the EndNote and BibTex citation formats. These changes correct
+errors and improve conformance with best practices but could break
+parsing of these formats.
+
+For more information, see #10790.

doc/release-notes/10809-oai-ore-nested-compound.md

@@ -0,0 +1 @@
+The OAI-ORE exporter can now export metadata containing nested compound fields (i.e. compound fields within compound fields). See #10809 and #11190.

doc/release-notes/10975-fix-file-replace-via-api

@@ -0,0 +1,3 @@
+A bug that caused replacing files via API when file PIDs were enabled has been fixed.
+
+ For testing purposes, the FAKE PID provider can now be used with file PIDs enabled. (The FAKE provider is not recommended for any production use.)

doc/release-notes/11053-metadata-styling-fixes

@@ -0,0 +1 @@
+Minor styling fixes for the Related Publication Field and fields using ORCID or ROR have been made (see #11053, #10964, #11106)

doc/release-notes/11120-add-3d-object-metadata-block.md

@@ -0,0 +1,60 @@
+### New 3D Object Data Metadata Block
+
+A new metadata block has been added for describing 3D object data. You can download it from the [guides](https://dataverse-guide--11167.org.readthedocs.build/en/11167/user/appendix.html). See also #11120 and #11167.
+
+All new Dataverse installations will receive this metadata block by default. We recommend adding it by following the upgrade instructions below.
+
+## Upgrade Instructions
+
+### For 6.6-Release-notes.md
+
+6\. Restart Payara
+
+7\. Update metadata blocks
+
+These changes reflect incremental improvements made to the handling of core metadata fields.
+
+```shell
+wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/scripts/api/data/metadatablocks/citation.tsv
+
+curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv
+```
+```shell
+wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/scripts/api/data/metadatablocks/3d_objects.tsv
+
+curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file 3d_objects.tsv
+```
+
+8\. Update Solr schema.xml file. Start with the standard v6.6 schema.xml, then, if your installation uses any custom or experimental metadata blocks, update it to include the extra fields (step 8a).
+
+Stop Solr (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.6/installation/prerequisites.html#solr-init-script)).
+
+```shell
+service solr stop
+```
+
+Replace schema.xml
+
+```shell
+wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/conf/solr/schema.xml
+cp schema.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf
+```
+
+Start Solr (but if you use any custom metadata blocks or adding 3D Objects, perform the next step, 8a first).
+
+```shell
+service solr start
+```
+8a\. For installations with custom or experimental metadata blocks:
+
+Before starting Solr, update the schema to include all the extra metadata fields that your installation uses. We do this by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed to reflect the names of the directories, if different):
+
+```shell
+	wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/conf/solr/update-fields.sh
+	chmod +x update-fields.sh
+	curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml
+```
+
+Now start Solr.
+
+9\. Reindex Solr

doc/release-notes/11127-update-search-api-to-show-dvobject-type-counts.md

@@ -0,0 +1,5 @@
+### show-type-counts Behavior changed in Search API
+
+In the Search API if you set show_type_counts=true the response will include all object types (Dataverses, Datasets, and Files) even if the search result for any given type is 0.
+
+See also the [guides](https://preview.guides.gdcc.io/en/develop/api/search.html#parameters), #11127 and #11138.

doc/release-notes/11129-send-feedback-to-contacts.md

@@ -0,0 +1,14 @@
+This feature adds a new API to send feedback to the Collection, Dataset, or DataFile's contacts.
+Similar to the "admin/feedback" API the "sendfeedback" API sends an email to all the contacts listed for the Dataset. The main differences for this feature are:
+1. This API is not limited to Admins
+2. This API does not return the email addresses in the "toEmail" and "ccEmail" elements for privacy reasons
+3. This API can be rate limited to avoid spamming
+4. The body size limit can be configured
+5. The body will be stripped of any html code to prevent malicious scripts or links
+6. The fromEmail will be validated for correct format
+
+To set the Rate Limiting for guest users (See Rate Limiting Configuration for more details. This example allows 1 send per hour for any guest)
+``curl http://localhost:8080/api/admin/settings/:RateLimitingCapacityByTierAndAction -X PUT -d '[{\"tier\": 0, \"limitPerHour\": 1, \"actions\": [\"CheckRateLimitForDatasetFeedbackCommand\"]}]'``
+
+To set the message size limit (example limit of 1080 chars):
+``curl -X PUT -d 1080 http://localhost:8080/api/admin/settings/:ContactFeedbackMessageSizeLimit``

doc/release-notes/11136-solr-nested.md

@@ -0,0 +1 @@
+Deeply nested metadata fields are not supported but the code used to generate the Solr schema has been adjusted to support them. See #11136.

doc/release-notes/11142-more-detailed-file-differences.md

@@ -0,0 +1,3 @@
+The file page version table now shows more detail, e.g. when there are metadata changes or whether a file has been replaced. 
+A bug that causes adding free-form provenance to a file to fail has been fixed.
+See also #11142 and #11145.

doc/release-notes/11159-preview-url-update.md

@@ -0,0 +1,7 @@
+### Preview URL popup updated
+
+The Preview URL popup and related documentation has been updated to give the dataset more information about anonymous access including the names of the dataset fields that will be withheld from the url user and suggesting how to review the url before releasing it. See also #11159 and #11164.
+
+###Bug Fix
+
+Bug which causes users of the Anonymous Review URL to have some metadata of published datasets withheld has been fixed. See #11202

doc/release-notes/11256-news-on-support-for-external-vocabulary.md

@@ -0,0 +1,4 @@
+### News on Support for External Vocabulary Services
+
+It is now possible to fill Dataverse keywords metadata using an OntoPortal service.
+The code has been shared on [GDCC GitHub Repository](https://github.com/gdcc/dataverse-external-vocab-support#scripts-in-production).

doc/release-notes/6467-optimize-permission-lookups-for-a-user.md

@@ -0,0 +1,9 @@
+The following API have been added:
+
+/api/users/{identifier}/allowedCollections/{permission}
+
+This API lists the dataverses/collections that the user has access to via the permission passed.
+By passing "any" as the permission the list will return all dataverse/collections that the user can access regardless of which permission is used.
+This API can be executed only by the User requesting their own list of accessible collections or by an Administrator.
+Valid Permissions are: AddDataverse, AddDataset, ViewUnpublishedDataverse, ViewUnpublishedDataset, DownloadFile, EditDataverse, EditDataset, ManageDataversePermissions,
+ManageDatasetPermissions, ManageFilePermissions, PublishDataverse, PublishDataset, DeleteDataverse, DeleteDatasetDraft, and "any" as a wildcard option.

doc/release-notes/guides-toc-depth.md

@@ -0,0 +1,3 @@
+### Guides Table of Contents Depth
+
+On the home page for each guide (User Guide, etc.) there was an overwhelming amount of information in the form of a deeply nested tabled of contents. The depth of the table of contents has been reduced to two levels, making the home page for each guide more readable. See #11166.

doc/release-notes/openaire_fix.md

@@ -0,0 +1 @@
+bugfix: openaire implementation can now correctly process one or multiple productionPlaces as geolocation

doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld

@@ -1,11 +1,31 @@
 {
 "citation:depositor": "Admin, Dataverse",
 "title": "Test Dataset",
+"socialscience:collectionMode": [
+  "demonstration"
+],
 "subject": "Computer and Information Science",
+"geospatial:geographicCoverage": [
+  {
+    "geospatial:otherGeographicCoverage": "Cambridge"
+  },
+  {
+    "geospatial:otherGeographicCoverage": "Massachusetts"
+  }
+],
 "author": {
   "citation:authorName": "Admin, Dataverse",
   "citation:authorAffiliation": "GDCC"
 },
+"kindOfData": "demonstration data",
+"citation:keyword": [
+  {
+    "citation:keywordValue": "first keyword"
+  },
+  {
+    "citation:keywordValue": "second keyword"
+  }
+],
 "dateOfDeposit": "2020-10-08",
 "citation:distributor": {
   "citation:distributorName": "Demo Dataverse Repository",
@@ -35,5 +55,9 @@
   "title": "http://purl.org/dc/terms/title",
   "citation": "https://dataverse.org/schema/citation/",
   "dvcore": "https://dataverse.org/schema/core#",
-  "schema": "http://schema.org/"
-}}
+  "schema": "http://schema.org/",
+  "geospatial": "dataverse.siteUrl/schema/geospatial#",
+  "socialscience": "dataverse.siteUrl/schema/socialscience#",
+  "kindOfData": "http://rdf-vocabulary.ddialliance.org/discovery#kindOfData"
+  }
+}

doc/sphinx-guides/source/_static/api/transform-oai-ore-jsonld.xq

@@ -0,0 +1,16 @@
+declare option output:method "json";
+
+let $parameters:={ 'method': 'json' }
+for $record in /json
+  let $metadata:=$record/ore_003adescribes
+
+
+  let $json:=
+    <json type="object">
+      {$metadata/*}
+      {$record/_0040context}
+    </json>
+
+
+  return if ($metadata) then
+      file:write("converted.json",$json, $parameters)

doc/sphinx-guides/source/admin/discoverability.rst

@@ -51,7 +51,7 @@ The Dataverse team has been working with Google on both formats. Google has `ind
 Signposting
 +++++++++++
 
-The Dataverse software supports `Signposting <https://signposting.org>`_. This allows machines to request more information about a dataset through the `Link <https://tools.ietf.org/html/rfc5988>`_ HTTP header.
+The Dataverse software supports `Signposting <https://signposting.org>`_. This allows machines to request more information about a dataset through the `Link <https://tools.ietf.org/html/rfc5988>`_ HTTP header. Links to all enabled metadata export formats are given. See :ref:`metadata-export-formats` for a list.
 
 There are 2 Signposting profile levels, level 1 and level 2. In this implementation, 
  * Level 1 links are shown `as recommended <https://signposting.org/FAIR/>`_ in the "Link"

doc/sphinx-guides/source/admin/index.rst

@@ -11,6 +11,7 @@ This guide documents the functionality only available to superusers (such as "da
 **Contents:**
 
 .. toctree::
+   :maxdepth: 2
 
    dashboard
    external-tools

doc/sphinx-guides/source/api/changelog.rst

@@ -12,6 +12,7 @@ v6.6
 
 - **/api/metadatablocks** is no longer returning duplicated metadata properties and does not omit metadata properties when called.
 - **/api/roles**: :ref:`show-role` now properly returns 403 Forbidden instead of 401 Unauthorized when you pass a working API token that doesn't have the right permission.
+- The content type for the ``schema.org`` dataset metadata export format has been corrected. It was ``application/json`` and now it is ``application/ld+json``. See also :ref:`export-dataset-metadata-api`.
 
 v6.5
 ----

doc/sphinx-guides/source/api/index.rst

@@ -9,6 +9,7 @@ API Guide
 **Contents:**
 
 .. toctree::
+   :maxdepth: 2
 
    intro
    getting-started
@@ -24,4 +25,4 @@ API Guide
    linkeddatanotification
    apps
    faq
-   changelog
+   changelog