diff --git a/build/readme.md b/build/readme.md index 96cee65c..62cfc742 100644 --- a/build/readme.md +++ b/build/readme.md @@ -46,7 +46,8 @@ Install software: $ pip install pyinstaller $ brew install upx -The following steps can alternatively be executed with build/osx/build.pyinstaller.command. Double click build.pyinstaller.command in Finder or execute in terminal. +The following steps can alternatively be executed with build/osx/build.pyinstaller.command. +Double click build.pyinstaller.command in Finder or execute in terminal. - Copy Facepager.spec to src folder: $ cp ../build/osx/Facepager.spec Facepager.spec diff --git a/build/windows/build.bat b/build/windows/build.bat index 2ee10bb3..73ddeec7 100644 --- a/build/windows/build.bat +++ b/build/windows/build.bat @@ -15,6 +15,6 @@ copy ..\..\build\windows\Facepager_Setupscript.nsi Facepager_Setupscript.nsi copy ..\..\icons\icon_facepager.ico icon_facepager.ico "C:\Program Files (x86)\NSIS\makensis.exe" Facepager_Setupscript.nsi copy Facepager_Setup.exe ..\..\build\windows\Facepager_Setup.exe - +cd ../../build/windows endlocal pause \ No newline at end of file diff --git a/presets/Bibliometrics-COCI_Metadata_from_doi.fp4.json b/presets/Bibliometrics-COCI_Metadata_from_doi.fp4.json deleted file mode 100644 index 75ea0a50..00000000 --- a/presets/Bibliometrics-COCI_Metadata_from_doi.fp4.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "name": "COCI Metadata from doi", - "category": "Bibliometrics", - "description": "Add DOIs as seed nodes, e.g. \"10.1111/j.1460-2466.2008.01401.x\". \n\n\n\nhttps://opencitations.net/index/coci/api/v1#/metadata/{dois}", - "module": "Generic", - "options": { - "basepath": "https://w3id.org/oc/index/coci/api/v1", - "resource": "/metadata/", - "params": { - "": "" - }, - "extension": "", - "headers": {}, - "verb": "GET", - "format": "json", - "filename": "", - "fileext": "", - "pages": 1, - "paging_type": "count", - "param_paging": null, - "offset_start": 1, - "offset_step": 1, - "nodedata": null, - "objectid": "doi", - "scope": "", - "proxy": "", - "auth_type": "Disable", - "auth_uri": "", - "redirect_uri": "", - "token_uri": "", - "auth": "disable", - "auth_tokenname": "", - "auth_prefix": "" - }, - "speed": 200, - "saveheaders": false, - "timeout": 15, - "maxsize": 5, - "columns": [ - "year", - "author", - "reference", - "reference_count=reference|re:^.\\|;|length", - "citation", - "citation_count", - "title", - "source_title", - "source_id", - "volume", - "issue", - "page", - "oa_link" - ] -} \ No newline at end of file diff --git a/presets/Bibliometrics-COCI_citations.fp4.json b/presets/Bibliometrics-COCI_citations.fp4.json deleted file mode 100644 index e1858a27..00000000 --- a/presets/Bibliometrics-COCI_citations.fp4.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name": "COCI: citations", - "category": "Bibliometrics", - "description": "Add DOIs as seed nodes, e.g. \"10.1111/j.1460-2466.2008.01401.x\". Gets the titles that cite the seed work. \n\nEach item is a citation record. To get bibliographic data use the metadata preset.\n\nSee https://opencitations.net/index/coci/api/v1#/citations/{doi}\n\n", - "module": "Generic", - "options": { - "auth": "disable", - "auth_prefix": "", - "auth_tokenname": "", - "auth_type": "Disable", - "auth_uri": "", - "basepath": "https://w3id.org/oc/index/coci/api/v1", - "extension": "", - "fileext": "", - "filename": "", - "format": "json", - "headers": {}, - "nodedata": null, - "objectid": "citing", - "offset_start": 1, - "offset_step": 1, - "pages": 1, - "paging_type": "count", - "param_paging": null, - "params": { - "": "" - }, - "proxy": "", - "redirect_uri": "", - "resource": "/citations/", - "scope": "", - "token_uri": "", - "verb": "GET" - }, - "speed": 200, - "saveheaders": false, - "timeout": 15, - "maxsize": 5, - "columns": [ - "creation", - "timespan", - "author_sc", - "journal_sc", - "oci", - "citing", - "cited" - ] -} \ No newline at end of file diff --git a/presets/Bibliometrics-COCI_references.fp4.json b/presets/Bibliometrics-COCI_references.fp4.json deleted file mode 100644 index 3da4d0d4..00000000 --- a/presets/Bibliometrics-COCI_references.fp4.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name": "COCI: references", - "category": "Bibliometrics", - "description": "Add DOIs as seed nodes, e.g. \"10.1111/j.1460-2466.2008.01401.x\". \n\nGets the references contained in the seed title. \n\nEach item is a citation record. To get bibliographic date use the metadata preset.\n\nSee https://opencitations.net/index/coci/api/v1#/references/{doi}\n\n", - "module": "Generic", - "options": { - "auth": "disable", - "auth_prefix": "", - "auth_tokenname": "", - "auth_type": "Disable", - "auth_uri": "", - "basepath": "https://w3id.org/oc/index/coci/api/v1", - "extension": "", - "fileext": "", - "filename": "", - "format": "json", - "headers": {}, - "nodedata": null, - "objectid": "cited", - "offset_start": 1, - "offset_step": 1, - "pages": 1, - "paging_type": "count", - "param_paging": null, - "params": { - "": "" - }, - "proxy": "", - "redirect_uri": "", - "resource": "/references/", - "scope": "", - "token_uri": "", - "verb": "GET" - }, - "speed": 200, - "saveheaders": false, - "timeout": 15, - "maxsize": 5, - "columns": [ - "creation", - "timespan", - "author_sc", - "journal_sc", - "oci", - "citing", - "cited" - ] -} \ No newline at end of file diff --git a/presets/Bibliometrics-OpenCitations_Index_COCI_Metadata_from_doi.fp4.json b/presets/Bibliometrics-OpenCitations_Index_COCI_Metadata_from_doi.fp4.json new file mode 100644 index 00000000..fea6de39 --- /dev/null +++ b/presets/Bibliometrics-OpenCitations_Index_COCI_Metadata_from_doi.fp4.json @@ -0,0 +1,54 @@ +{ + "name": "OpenCitations Index (COCI): Metadata from doi", + "category": "Bibliometrics", + "description": "[The OpenCitations Index](https://opencitations.net/index) – formerly known as the OpenCitations Index of Crossref open DOI-to-DOI citations (COCI) – is a bibliographic index recording and storing citations between publications. It pulls its data from various open scholarly sources as described on their website (see link above), thereby providing comprehensive and openly accessible bibliographic and citation data. \n\nEach citation is identified by an unique [Open Citation Identifier (OCI)](https://identifiers.org/oci). However, data can be fetched via various identifiers including DOIs. \n\nApply this preset to fetch metadate releated to a scholarly work. Add as many DOIs as seed nodes as you wish (e.g. , \"10.1111/j.1460-2466.2008.01401.x\").\n\nFor this specific preset we are still calling version 1.4.0 of the [REST API for COCI](https://opencitations.net/index/coci/api/v1#/metadata/%7Bdois%7D). As the operation \"/metadata/{dois}\" has not been carried to version 2.0.x of [the REST API for the OpenCitations Index](https://opencitations.net/index/api/v2). In this context OpenCitations states: \"we use certain legacy APIs for historical and compatibility reasons. While we encourage the use of our latest and more efficient V2 APIs, we understand that some applications may still rely on these older endpoints. Please note that legacy APIs may not receive updates or support, and their use may be phased out in the future. We recommend transitioning to our current V2 APIs for improved performance, security, and reliability.\"", + "module": "Generic", + "options": { + "auth": "disable", + "auth_prefix": "", + "auth_tokenname": "", + "auth_type": "Disable", + "auth_uri": "", + "basepath": "https://opencitations.net/index/coci/api/v1", + "extension": "", + "fileext": "", + "filename": "", + "format": "json", + "headers": {}, + "nodedata": null, + "objectid": "doi", + "offset_start": 1, + "offset_step": 1, + "pages": 1, + "paging_type": "count", + "param_paging": null, + "params": { + "": "" + }, + "proxy": "", + "redirect_uri": "", + "resource": "/metadata/", + "scope": "", + "token_uri": "", + "verb": "GET" + }, + "speed": 200, + "saveheaders": false, + "timeout": 15, + "maxsize": 5, + "columns": [ + "year", + "author", + "reference", + "reference_count=reference|re:^.\\|;|length", + "citation", + "citation_count", + "title", + "source_title", + "source_id", + "volume", + "issue", + "page", + "oa_link" + ] +} \ No newline at end of file diff --git a/presets/Bibliometrics-OpenCitations_Index_COCI_citations.fp4.json b/presets/Bibliometrics-OpenCitations_Index_COCI_citations.fp4.json new file mode 100644 index 00000000..670d4b5b --- /dev/null +++ b/presets/Bibliometrics-OpenCitations_Index_COCI_citations.fp4.json @@ -0,0 +1,50 @@ +{ + "name": "OpenCitations Index (COCI): citations", + "category": "Bibliometrics", + "description": "[The OpenCitations Index](https://opencitations.net/index) – formerly known as the OpenCitations Index of Crossref open DOI-to-DOI citations (COCI) – is a bibliographic index recording and storing citations between publications. It pulls its data from various open scholarly sources as described on their website (see link above), thereby providing comprehensive and openly accessible bibliographic and citation data. \n\nEach citation is identified by an unique [Open Citation Identifier (OCI)](https://identifiers.org/oci). However, data can be fetched via various identifiers including DOIs. Further, each item is a citation record. To get its bibliographic metadata use the \"OpenCitations Index (COCI): Metadata from doi\" preset.\n\nApply this preset to fetch the titles of all works that cite the seed work. Add as many DOIs as seed nodes as you wish (e.g. , \"10.1111/j.1460-2466.2008.01401.x\").\n\nHere we are calling version 2.0.x of [the REST API for the OpenCitations Index](https://opencitations.net/index/api/v2#/citations/%7Bid%7D).", + "module": "Generic", + "options": { + "auth": "disable", + "auth_prefix": "", + "auth_tokenname": "", + "auth_type": "Disable", + "auth_uri": "", + "basepath": "https://opencitations.net/index/api/v2", + "extension": "", + "fileext": "", + "filename": "", + "format": "json", + "headers": {}, + "nodedata": null, + "objectid": "citing", + "offset_start": 1, + "offset_step": 1, + "pages": 1, + "paging_type": "count", + "param_paging": null, + "params": { + "": "" + }, + "proxy": "", + "redirect_uri": "", + "resource": "/citations/doi:", + "scope": "", + "token_uri": "", + "verb": "GET" + }, + "speed": 200, + "saveheaders": false, + "timeout": 15, + "maxsize": 5, + "columns": [ + "oci", + "citing", + "cited", + "creation", + "timespan", + "journal_sc", + "author_sc", + "", + "" + ] +} \ No newline at end of file diff --git a/presets/Bibliometrics-OpenCitations_Index_COCI_references.fp4.json b/presets/Bibliometrics-OpenCitations_Index_COCI_references.fp4.json new file mode 100644 index 00000000..d80c3594 --- /dev/null +++ b/presets/Bibliometrics-OpenCitations_Index_COCI_references.fp4.json @@ -0,0 +1,48 @@ +{ + "name": "OpenCitations Index (COCI): references", + "category": "Bibliometrics", + "description": "[The OpenCitations Index](https://opencitations.net/index) – formerly known as the OpenCitations Index of Crossref open DOI-to-DOI citations (COCI) – is a bibliographic index recording and storing citations between publications. It pulls its data from various open scholarly sources as described on their website (see link above), thereby providing comprehensive and openly accessible bibliographic and citation data. \n\nEach citation is identified by an unique [Open Citation Identifier (OCI)](https://identifiers.org/oci). However, data can be fetched via various identifiers including DOIs. Further, each item is a citation record. To get its bibliographic metadata use the \"OpenCitations Index (COCI): Metadata from doi\" preset.\n\nApply this preset to fetch the references contained in the seed title. . Add as many DOIs as seed nodes as you wish (e.g. , \"10.1111/j.1460-2466.2008.01401.x\").\n\nHere we are calling version 2.0.x of [the REST API for the OpenCitations Index](https://opencitations.net/index/api/v2#/references/%7Bid%7D).", + "module": "Generic", + "options": { + "basepath": "https://opencitations.net/index/api/v2", + "resource": "/references/doi:", + "params": { + "": "" + }, + "extension": "", + "headers": {}, + "verb": "GET", + "format": "json", + "filename": "", + "fileext": "", + "pages": 1, + "paging_type": "count", + "param_paging": null, + "offset_start": 1, + "offset_step": 1, + "nodedata": null, + "objectid": "cited", + "scope": "", + "proxy": "", + "auth_type": "Disable", + "auth_uri": "", + "redirect_uri": "", + "token_uri": "", + "auth": "disable", + "auth_tokenname": "", + "auth_prefix": "" + }, + "speed": 200, + "saveheaders": false, + "timeout": 15, + "maxsize": 5, + "columns": [ + "creation", + "timespan", + "author_sc", + "journal_sc", + "oci", + "citing", + "cited" + ] +} \ No newline at end of file diff --git a/presets/Bibliometrics-Open_Library_Author_metadata_Authors_API_.fp4.json b/presets/Bibliometrics-Open_Library_Author_metadata_Authors_API_.fp4.json new file mode 100644 index 00000000..f536970b --- /dev/null +++ b/presets/Bibliometrics-Open_Library_Author_metadata_Authors_API_.fp4.json @@ -0,0 +1,45 @@ +{ + "name": "Open Library: Author metadata (Authors API)", + "category": "Bibliometrics", + "description": "[Open Library](https://openlibrary.org/) is an open project launched by the Internet Archive in 2006 that aims to provide \"one web page for every book ever published\". Each page contains detailed bibliographic information about the entry, including metadata such as publisher, publication date, and ISBN. The result is an exhaustive catalog of books searchable by title, author, or subject and, whenever possible, Open Library offers digital versions of its books for borrowing and reading. Due to its collaborative, wiki-style nature all users are welcome to contribute information to the catalog by editing book information or even adding new entries.\n\nApply this preset to fetch all available metadata about an author using the [Open Library Author API](https://openlibrary.org/dev/docs/api/authors). Start by adding an author's identifier key as seed note (e.g., \"OL234664A\"). You can also fetch information by adding the name of an author as a seed node (e.g., \"George R.R. Martin\"). Simply, use the Open Library Search API by applying our \"Open Library: Book metadata from title (Search API)\" preset and replace \"title:\" in the q parameter with \"author:\". Just make sure to adjust the Column Setup accordingly.\n\nPlease, see this [overview of Open Library's range of APIs](https://openlibrary.org/developers/api) to find the API that best suits your needs and adjust the preset accordingly. Generally, OpenLibrary recommends their powerful [Search API](https://openlibrary.org/dev/docs/api/search) and they have published some [tips for working with it](https://openlibrary.org/search/howto).\n\n**Note:** Please mind Open Library's bulk download policy. Open Library kindly asks to not use their APIs for bulk downloads. If you need a dump of complete data, please read about their [bulk download](https://openlibrary.org/data#downloads) options, or email them at openlibrary@archive.org. If you plan to make regular, frequent use of Open Library's APIs (e.g. multiple calls per minute), please edit the existing **header** to specify the **User-Agent string** with (a) the name of your application and (b) your contact email or phone number, so Open Library may contact you when we notice high request volume. Failing to include these headers may result in the application (Facepager) being blocked.", + "module": "Generic", + "options": { + "auth": "disable", + "auth_prefix": "", + "auth_tokenname": "", + "auth_type": "Disable", + "auth_uri": "", + "basepath": "https://openlibrary.org", + "extension": "", + "fileext": "", + "filename": "", + "format": "json", + "headers": { + "User-Agent": "Facepager/4.5.3 (youremail@example.com)" + }, + "nodedata": null, + "objectid": "personal_name", + "offset_start": 1, + "offset_step": 1, + "pages": 1, + "paging_type": "count", + "param_paging": null, + "params": { + "": "" + }, + "proxy": "", + "redirect_uri": "", + "resource": "/authors/.json", + "scope": "", + "token_uri": "", + "verb": "GET" + }, + "speed": 200, + "saveheaders": false, + "timeout": 15, + "maxsize": 5, + "columns": [ + "birth_date", + "bio" + ] +} \ No newline at end of file diff --git a/presets/Bibliometrics-Open_Library_Book_metadata_from_title_Search_API_.fp4.json b/presets/Bibliometrics-Open_Library_Book_metadata_from_title_Search_API_.fp4.json new file mode 100644 index 00000000..ee27f4b7 --- /dev/null +++ b/presets/Bibliometrics-Open_Library_Book_metadata_from_title_Search_API_.fp4.json @@ -0,0 +1,55 @@ +{ + "name": "Open Library: Book metadata from title (Search API)", + "category": "Bibliometrics", + "description": "[Open Library](https://openlibrary.org/) is an open project launched by the Internet Archive in 2006 that aims to provide \"one web page for every book ever published\". Each page contains detailed bibliographic information about the entry, including metadata such as publisher, publication date, and ISBN. The result is an exhaustive catalog of books searchable by title, author, or subject and, whenever possible, Open Library offers digital versions of its books for borrowing and reading. Due to its collaborative, wiki-style nature all users are welcome to contribute information to the catalog by editing book information or even adding new entries.\n\nApply this preset to fetch all available metadata about a book or a book series using the [Open Library Search API](https://openlibrary.org/dev/docs/api/search). Start by adding a book title as seed note (e.g., \"A Song of Ice and Fire\"). This will return all works that contain the seed need in their title. Edit the fields parameter to specifiy the metadata returned by your query. \n\nPlease, see this [overview of Open Library's range of APIs](https://openlibrary.org/developers/api) to find the API that best suits your needs and adjust the preset accordingly. Generally, OpenLibrary recommends their powerful [Search API](https://openlibrary.org/dev/docs/api/search) and they have published some [tips for working with it](https://openlibrary.org/search/howto).\n\n**Note:** Please mind Open Library's bulk download policy. Open Library kindly asks to not use their APIs for bulk downloads. If you need a dump of complete data, please read about their [bulk download](https://openlibrary.org/data#downloads) options, or email them at openlibrary@archive.org. If you plan to make regular, frequent use of Open Library's APIs (e.g. multiple calls per minute), please edit the existing **header** to specify the **User-Agent string** with (a) the name of your application and (b) your contact email or phone number, so Open Library may contact you when we notice high request volume. Failing to include these headers may result in the application (Facepager) being blocked.", + "module": "Generic", + "options": { + "auth": "disable", + "auth_prefix": "", + "auth_tokenname": "", + "auth_type": "Disable", + "auth_uri": "", + "basepath": "https://openlibrary.org", + "extension": "", + "fileext": "", + "filename": "", + "format": "json", + "headers": { + "User-Agent": "Facepager/4.5.3 (youremail@example.com)" + }, + "nodedata": "docs", + "objectid": "author_name.0", + "offset_start": 1, + "offset_step": 1, + "pages": 1, + "paging_type": "count", + "param_paging": null, + "params": { + "fields": "*", + "limit": "10", + "q": "title:", + "sort": "old" + }, + "proxy": "", + "redirect_uri": "", + "resource": "/search.json?", + "scope": "", + "token_uri": "", + "verb": "GET" + }, + "speed": 200, + "saveheaders": false, + "timeout": 15, + "maxsize": 5, + "columns": [ + "title", + "key", + "first_publish_year", + "author_name.0", + "author_key.0", + "edition_count", + "ebook_access", + "isbn.0", + "ratings_average" + ] +} \ No newline at end of file diff --git a/presets/Generic-DBPedia-Scientists_who_have_birthday_today.3_9.json b/presets/KnowledgeGraph-DBPedia-Scientists_who_have_birthday_today.3_9.json similarity index 98% rename from presets/Generic-DBPedia-Scientists_who_have_birthday_today.3_9.json rename to presets/KnowledgeGraph-DBPedia-Scientists_who_have_birthday_today.3_9.json index 8b621015..60f5e4f2 100644 --- a/presets/Generic-DBPedia-Scientists_who_have_birthday_today.3_9.json +++ b/presets/KnowledgeGraph-DBPedia-Scientists_who_have_birthday_today.3_9.json @@ -1,7 +1,7 @@ { "description": "Get the names, descriptions, birthdays and if available thumbnail images of scientists who have birthday today. The results are limited to English and to the 10 oldest scientists. This preset is compatible with Facepager v3.9.2 or newer. \n\nAdd an arbitrary node and fetch data. The node name doesn't matter as it is not used in the query. Suggestion: add dates as node names and adjust the query to get scientists who have birthday at a specific date.\n\nDBPedia is like Wikidata a semantic web version of Wikipedia. On the semantic web data is organizied according to the Resource Decription Framework (RDF). The preset retrieves data from the SPARQL endpoint of DBPedia. DBPedia mainly consists of content scrapted from the info boxes of Wikipedia. With limiting to english names and descriptions the results should reflect the English Wikipedia.\n\nSee https://wiki.dbpedia.org/ for details about DBPedia. If you are not familiar with semantic web technology consider reading Wikipedia articles about the Resource Description Framework and about SPARQL.\n\nThe example query was taken from an OpenHPI course: https://open.hpi.de/courses/semanticweb2017/\n\nPay attention on how the SPARQL query is formulated. Since angle brackets signify placeholders in Facepager they have to be masked with a backslash. If you need a backslash in your query, mask it by another backslash. To use the name of a node or data of previously fetched nodes in your query user placeholders (see the Facepager help).\n", "module": "Generic", - "category": "DBPedia", + "category": "Knowledge Graph", "speed": 200, "options": { "resource": "/sparql", diff --git a/presets/NFDI4Culture-1_Get_GND_IDs_of_Ferdinand_Gregorovius_addressees.fp4.json b/presets/KnowledgeGraph-NFDI4Culture-1_Get_GND_IDs_of_Ferdinand_Gregorovius_addressees.fp4.json similarity index 90% rename from presets/NFDI4Culture-1_Get_GND_IDs_of_Ferdinand_Gregorovius_addressees.fp4.json rename to presets/KnowledgeGraph-NFDI4Culture-1_Get_GND_IDs_of_Ferdinand_Gregorovius_addressees.fp4.json index 3e3341e8..4013df86 100644 --- a/presets/NFDI4Culture-1_Get_GND_IDs_of_Ferdinand_Gregorovius_addressees.fp4.json +++ b/presets/KnowledgeGraph-NFDI4Culture-1_Get_GND_IDs_of_Ferdinand_Gregorovius_addressees.fp4.json @@ -1,6 +1,6 @@ { - "name": "1 Get GND IDs of Ferdinand Gregorovius' addressees", - "category": "NFDI4Culture", + "name": "NFDI4Culture: 1 Get GND IDs of Ferdinand Gregorovius' addressees", + "category": "Knowledge Graph", "description": "This preset accompanies our [Getting Started with Culture Knowledge Graph](https://github.com/strohne/Facepager/wiki/Getting-Started-with-CultureKnowledgeGraph) guide. \n\nIt retrives GND IDs of Ferdinand Gregorovius' addressees based on years as seed nodes. At the heart of the preset lies a SPARQL query. Making an effort to learn the basics of SPARQL's syntax will allow you to literally understand any such query and, more crucially, let's you write your own queries, thus empowering you to answer (research) questions of your interest. A great resources to start with is, for example, [Wikidata's introduction to SPARQL](https://www.wikidata.org/wiki/Wikidata:SPARQL_tutorial). For now, it will suffice to understand that the query returns the url, label, and date of any letter Gregorovius wrote during the specfied year as well as the GND IDs of all persons addressed or mention in each letter.\n\n**DISCLAIMER:** The Culture Knowledge Graph is work in progress. Currently, only a handful of datasets are accessible through its SPARQL endpoint using the NFDI4Culture Ontology. The User-Policy and guidelines of the Culture Knowledge Graph are being worked on as well. For the time being we advise you to be mindful of potential query limits.", "module": "Generic", "options": { @@ -15,7 +15,7 @@ "verb": "POST", "format": "json", "encoding": "", - "payload": "PREFIX cto: \\\nPREFIX schema: \\\nPREFIX rdf: \\\nPREFIX rdfs: \\\nPREFIX n4c: \\\n\nSELECT ?letter ?letterLabel ?date ?gnd_id\nWHERE {\n n4c:E5378 schema:dataFeedElement/schema:item ?letter .\n ?letter rdfs:label ?letterLabel ;\n cto:creationDate ?date ;\n cto:gnd ?gnd_url .\n FILTER(YEAR(?date) = ).\n BIND(REPLACE(STR(?gnd_url), \".*://.*/(.*?)\", \"$1\") AS ?gnd_id)\n}", + "payload": "PREFIX cto: \\\nPREFIX schema: \\\nPREFIX rdf: \\\nPREFIX rdfs: \\\nPREFIX n4c: \\\n\nSELECT ?letter ?letterLabel ?date ?gnd_id\nWHERE {\n n4c:E5378 schema:dataFeedElement/schema:item ?letter .\n ?letter rdfs:label ?letterLabel ;\n cto:creationDate ?date ;\n cto:relatedPerson ?gnd_url .\n FILTER(YEAR(?date) = ).\n BIND(REPLACE(STR(?gnd_url), \".*://.*/(.*?)\", \"$1\") AS ?gnd_id)\n}", "filename": "", "fileext": "", "pages": 1, diff --git a/presets/NFDI4Culture-2_Translate_GND_ID_to_a_person_s_name.fp4.json b/presets/KnowledgeGraph-NFDI4Culture-2_Translate_GND_ID_to_a_person_s_name.fp4.json similarity index 92% rename from presets/NFDI4Culture-2_Translate_GND_ID_to_a_person_s_name.fp4.json rename to presets/KnowledgeGraph-NFDI4Culture-2_Translate_GND_ID_to_a_person_s_name.fp4.json index 16c282b2..21df116e 100644 --- a/presets/NFDI4Culture-2_Translate_GND_ID_to_a_person_s_name.fp4.json +++ b/presets/KnowledgeGraph-NFDI4Culture-2_Translate_GND_ID_to_a_person_s_name.fp4.json @@ -1,6 +1,6 @@ { - "name": "2 Translate GND ID to a person's name", - "category": "NFDI4Culture", + "name": "NFDI4Culture: 2 Translate GND ID to a person's name (Entity Facts API; German National Library)", + "category": "Knowledge Graph", "description": "This preset accompanies our [Getting Started with Culture Knowledge Graph](https://github.com/strohne/Facepager/wiki/Getting-Started-with-CultureKnowledgeGraph) guide. \n\nIt translates the GND IDs of Ferdinand Gregorovius' addressees from the first preset into the persons' preferred names (from a historic perspective) for further use. Generally, you can use this preset to obtain any data accessible through the [Entity Facts API](https://www.dnb.de/EN/Professionell/Metadatendienste/Datenbezug/Entity-Facts/entityFacts_node.html#doc250704bodyText8) of the German National Library. To get an overview of the possibilities, please, refer to the [library's online sheet](https://wiki.dnb.de/pages/viewpage.action?pageId=134055670).\n\nPlease, mind the German National Library's [Terms of Use](https://www.dnb.de/EN/Professionell/Metadatendienste/Datenbezug/geschaeftsmodell.html).", "module": "Generic", "options": { diff --git a/presets/KnowledgeGraph-SemOpenAlex_Get_a_single_work.fp4.json b/presets/KnowledgeGraph-SemOpenAlex_Get_a_single_work.fp4.json new file mode 100644 index 00000000..bc770fda --- /dev/null +++ b/presets/KnowledgeGraph-SemOpenAlex_Get_a_single_work.fp4.json @@ -0,0 +1,48 @@ +{ + "name": "SemOpenAlex: Get a single work", + "category": "Knowledge Graph", + "description": "This SemOpenAlex preset poses an alternative to fetching data from OpenAlex's API using SPARQL instead. In terms of what information will be fetched, it mirrors the \"OpenAlex 1: Get a single work\" preset (not all information is readily available yet, see the SPARQL query for details).\n\n[SemOpenAlex](https://semopenalex.org/resource/semopenalex:UniversalSearch) is built upon [OpenAlex](https://openalex.org/about) and contains the same comprehensive information on scholarly works and related entities, such as authors, sources, etc.. It can practically be understood as a semantic extension of OpenAlex and aims at making the shared data base more accessible for use in semantic web technologies and linked open data applications. For this purpose, it offers an URI Resolution as well as a [SPARQL endpoint](https://semopenalex.org/sparql) available through the [SemOpenAlex Ontology](https://semopenalex.org/resource/?uri=https://semopenalex.org/ontology/).\n\nApply this preset if you are interested in using SemOpenAlex's SPARQL endpoint to retrieve a range of information about scholarly works based on its title (e.g., \"Computational Methods f\u00fcr die Sozial- und Geisteswissenschaften\"). Although featured in SemOpenAlex's Ontology, finding scholarly works by their DOI does not work yet. Should no result return, always check if any of the data you are intereseted in is not available on SemOpenAlex. For other (more complex) queries, please, see [SemOpenAlex's SPARQL examples](https://semopenalex.org/resource/semopenalex:About).\n\nTo learn more about the basic usage of SPARQL and its semantic structure, please, see our [Getting Started with Wikidata](https://github.com/strohne/Facepager/wiki/Getting-Started-with-Wikidata) where we dedicated a whole chapter to a SPARQL introduction. Or have a look into [SemOpenAlex's own documentation](https://semopenalex.org/resource/Help:WorkingWithData#querying-retrieving-data).", + "module": "Generic", + "options": { + "auth": "disable", + "auth_prefix": "", + "auth_tokenname": "", + "auth_type": "Disable", + "auth_uri": "", + "basepath": "https://semopenalex.org", + "extension": "", + "fileext": "", + "filename": "", + "format": "json", + "headers": { + "Accept": "application/sparql-results+json", + "Content-Type": "application/sparql-query" + }, + "key_paging": null, + "nodedata": "results.bindings", + "objectid": null, + "pages": 1, + "paging_stop": null, + "paging_type": "key", + "param_paging": null, + "params": { + "query": "PREFIX rdf: \\ \nPREFIX dct: \\\nPREFIX xsd: \\\nPREFIX prism: \\\nPREFIX fabio: \\\nPREFIX cito: \\\nPREFIX soa: \\\nPREFIX mplabel: \\\nPREFIX foaf: \\\n\nSELECT ?work ?title ?doi ?year ?references ?author ?authoruri WHERE {\n\n # Defining the class of our main subject\n ?work rdf:type \\.\n\n # Setting up a placeholder\n ?work dct:title \"\";\n\n # Fetching the title (only useful if resolving a DOI)\n #dct:title ?title;\n\n # Although featured in SemOpenAlex's ontology, fetching or resolving DOIs of scholarly works is not yet available\n #prism:doi ?doi;\n\n fabio:hasPublicationYear ?year;\n\n # If any of the requested data is not available, no results will be shown,\n # here for example references are not stored. This is entity dependend!\n #cito:cites ?references;\n\n dct:creator ?authoruri.\n ?authoruri foaf:name ?author.\n\n}" + }, + "proxy": "", + "redirect_uri": "", + "resource": "/sparql", + "scope": "", + "token_uri": "", + "verb": "GET" + }, + "speed": 200, + "saveheaders": false, + "timeout": 15, + "maxsize": 5, + "columns": [ + "work.value", + "year.value", + "author.value", + "authoruri.value" + ] +} \ No newline at end of file diff --git a/presets/Wikidata-Get_gender_of_solo_artist_band_members.fp4.json b/presets/KnowledgeGraph-Wikidata-Get_gender_of_solo_artist_band_members.fp4.json similarity index 96% rename from presets/Wikidata-Get_gender_of_solo_artist_band_members.fp4.json rename to presets/KnowledgeGraph-Wikidata-Get_gender_of_solo_artist_band_members.fp4.json index 8f563d66..cc3cc63e 100644 --- a/presets/Wikidata-Get_gender_of_solo_artist_band_members.fp4.json +++ b/presets/KnowledgeGraph-Wikidata-Get_gender_of_solo_artist_band_members.fp4.json @@ -1,6 +1,6 @@ { - "name": "Get gender of solo artist/band members", - "category": "Wikidata", + "name": "Wikidata: Get gender of solo artist/band members", + "category": "Knowledge Graph", "description": "This preset fetches a list of muscial artists and their sex or gender as provided by Wikidata. To start add the English name (label) of an arbitary musical solo artist or band as seed node, e.g. \"Peter Maffay\" or \"Coldplay\". The query automatically extracts names of band members and their respective on the basis of the group's name, e.g. \"Coldplay\". \n\nSee https://www.wikidata.org/ for further information. You can try out different queries on https://query.wikidata.org/ (paste the value of the query parameter).\n\nAlways ensure to comply to Wikidata's User-Agent policy and be mindful of potential query limits.\n\nPlease be aware that expressions in angle brackets are handled as placeholder by Facepager. Thus, the brackets of the prefixes have to be escaped with a backslash.\n\nPagination is handled by the placeholder.", "module": "Generic", "options": { diff --git a/presets/Wikidata-Get_list_of_biblical_characters.fp4.json b/presets/KnowledgeGraph-Wikidata-Get_list_of_biblical_characters.fp4.json similarity index 95% rename from presets/Wikidata-Get_list_of_biblical_characters.fp4.json rename to presets/KnowledgeGraph-Wikidata-Get_list_of_biblical_characters.fp4.json index ef74bc65..c3d00065 100644 --- a/presets/Wikidata-Get_list_of_biblical_characters.fp4.json +++ b/presets/KnowledgeGraph-Wikidata-Get_list_of_biblical_characters.fp4.json @@ -1,6 +1,6 @@ { - "name": "Get list of biblical characters", - "category": "Wikidata", + "name": "Wikidata: Get list of biblical characters", + "category": "Knowledge Graph", "description": "This preset gets a list of biblical characters, along with the sections in which they occur, from WikiData. Add an arbitrary seed node and fetch data.\n\nSee https://www.wikidata.org/wiki/Wikidata:Lists/List_of_biblical_characters\n\nYou can try out different query on https://query.wikidata.org/ (paste the value of the query parameter to get started).\n\n\nPlease be aware that expressions in angle brackets are handled as placeholder by Facepager. Thus, if your query contains angle brackets, escape them with a backslash. Pagination is handled by the placeholder.", "module": "Generic", "options": { diff --git a/presets/Wikidata-Which_politicians_did_not_die_on_natural_causes_.fp4.json b/presets/KnowledgeGraph-Wikidata-Which_politicians_did_not_die_on_natural_causes_.fp4.json similarity index 95% rename from presets/Wikidata-Which_politicians_did_not_die_on_natural_causes_.fp4.json rename to presets/KnowledgeGraph-Wikidata-Which_politicians_did_not_die_on_natural_causes_.fp4.json index 6439bc15..c469e73b 100644 --- a/presets/Wikidata-Which_politicians_did_not_die_on_natural_causes_.fp4.json +++ b/presets/KnowledgeGraph-Wikidata-Which_politicians_did_not_die_on_natural_causes_.fp4.json @@ -1,6 +1,6 @@ { - "name": "Which politicians did not die on natural causes?", - "category": "Wikidata", + "name": "Wikidata: Which politicians did not die on natural causes?", + "category": "Knowledge Graph", "description": "This preset fetches a list of politicians from WikiData that did not die on natural causes. Add an arbitrary node as seed node, e.g. \"politicians\".\n\nSee https://www.wikidata.org/ for further information. You can try out different queries on https://query.wikidata.org/ (paste the value of the query parameter).\n\nPlease be aware that expressions in angle brackets are handled as placeholder by Facepager. Thus, the brackets of the prefixes have to be escaped with a backslash.\n\nPagination is handled by the placeholder.", "module": "Generic", "options": { diff --git a/presets/Wikidata-Writers_Network.fp4.json b/presets/KnowledgeGraph-Wikidata-Writers_Network.fp4.json similarity index 97% rename from presets/Wikidata-Writers_Network.fp4.json rename to presets/KnowledgeGraph-Wikidata-Writers_Network.fp4.json index c7f9a2b9..e74b6314 100644 --- a/presets/Wikidata-Writers_Network.fp4.json +++ b/presets/KnowledgeGraph-Wikidata-Writers_Network.fp4.json @@ -1,6 +1,6 @@ { - "name": "Writers Network", - "category": "Wikidata", + "name": "Wikidata: Writers Network", + "category": "Knowledge Graph", "description": "This preset accompanies the [Getting Started with Wikidata](https://github.com/strohne/Facepager/wiki/Getting-Started-with-Wikidata) tutorial. It fetches a list of German writers as well as cities relevant to them based on literary movements. To start add a Q-identifier of an arbitary literary movement as seed node, e.g. \"Q37068\" for romanticism.\n\nSee https://www.wikidata.org/ for further information. You can try out different queries on https://query.wikidata.org/ (paste the value of the query parameter).\n\nAlways ensure to comply to [Wikidata's User-Agent policy](https://foundation.wikimedia.org/wiki/Policy:User-Agent_policy) and be mindful of potential [query limits](https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Query_limits).\n\nPlease be aware that expressions in angle brackets are handled as placeholder by Facepager. Thus, the brackets of the prefixes have to be escaped with a backslash.\n\nPagination is handled by the placeholder.\n", "module": "Generic", "options": { diff --git a/presets/Knowledge_Graph-DBPedia_Scientists_who_have_birhtday_today_SPARQL_Module_.fp4.json b/presets/Knowledge_Graph-DBPedia_Scientists_who_have_birhtday_today_SPARQL_Module_.fp4.json new file mode 100644 index 00000000..c5232770 --- /dev/null +++ b/presets/Knowledge_Graph-DBPedia_Scientists_who_have_birhtday_today_SPARQL_Module_.fp4.json @@ -0,0 +1,44 @@ +{ + "name": " (SPARQL Module) DBPedia: Scientists who have birhtday today", + "category": "Knowledge Graph", + "description": "This preset requires at least Facepager v4.6 \n\nGet the names, descriptions, birthdays and if available thumbnail images of scientists who have birthday today. The results are limited to English and to the 10 oldest scientists. This preset is compatible with Facepager v3.9.2 or newer. \n\nAdd an arbitrary node and fetch data. The node name doesn't matter as it is not used in the query. Suggestion: add dates as node names and adjust the query to get scientists who have birthday at a specific date.\n\nDBPedia is like Wikidata a semantic web version of Wikipedia. On the semantic web data is organizied according to the Resource Decription Framework (RDF). The preset retrieves data from the SPARQL endpoint of DBPedia. DBPedia mainly consists of content scrapted from the info boxes of Wikipedia. With limiting to english names and descriptions the results should reflect the English Wikipedia.\n\nSee https://wiki.dbpedia.org/ for details about DBPedia. If you are not familiar with semantic web technology consider reading Wikipedia articles about the Resource Description Framework and about SPARQL.\n\nThe example query was taken from an OpenHPI course: https://open.hpi.de/courses/semanticweb2017/\n\nPay attention on how the SPARQL query is formulated. Since angle brackets signify placeholders in Facepager they have to be masked with a backslash. If you need a backslash in your query, mask it by another backslash. To use the name of a node or data of previously fetched nodes in your query user placeholders (see the Facepager help).\n", + "module": "SPARQL", + "options": { + "auth": "disable", + "auth_prefix": "", + "auth_tokenname": "", + "auth_type": "Disable", + "auth_uri": "", + "basepath": "http://dbpedia.org/sparql", + "extension": "", + "format": "json", + "headers": { + "User-Agent": "FACEPAGERBOT/4.5 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.5" + }, + "key_paging": null, + "nodedata": "results.bindings", + "objectid": null, + "offset_start": 1, + "offset_step": 1, + "paging_stop": null, + "paging_type": null, + "param_paging": null, + "params": { + "format": "json", + "query": "PREFIX rdf: \\\nPREFIX dbo: \\\nPREFIX rdfs: \\\nPREFIX dc: \\\n\nSelect distinct ?birthdate ?thumbnail ?scientist ?name ?description WHERE {\n?scientist rdf:type dbo:Scientist ;\n dbo:birthDate ?birthdate ;\n rdfs:label ?name ;\n rdfs:comment ?description \n FILTER ((lang(?name)=\"en\")&&(lang(?description)=\"en\")&&(STRLEN(STR(?birthdate))>6)&&(SUBSTR(STR(?birthdate),6)=SUBSTR(STR(bif:curdate('')),6))) .\n OPTIONAL { ?scientist dbo:thumbnail ?thumbnail . }\n} ORDER BY ?birthdate LIMIT 10" + }, + "redirect_uri": "", + "resource": "", + "token_uri": "" + }, + "speed": 200, + "saveheaders": false, + "timeout": 15, + "maxsize": 5, + "columns": [ + "name.value", + "description.value", + "birthdate.value", + "thumbnail.value" + ] +} \ No newline at end of file diff --git a/presets/Knowledge_Graph-NFDI4Culture_1_Get_GND_IDs_of_Ferdinand_Gregorovius_s_addressees_SPARQL_module_.fp4.json b/presets/Knowledge_Graph-NFDI4Culture_1_Get_GND_IDs_of_Ferdinand_Gregorovius_s_addressees_SPARQL_module_.fp4.json new file mode 100644 index 00000000..450fcb77 --- /dev/null +++ b/presets/Knowledge_Graph-NFDI4Culture_1_Get_GND_IDs_of_Ferdinand_Gregorovius_s_addressees_SPARQL_module_.fp4.json @@ -0,0 +1,43 @@ +{ + "name": "(SPARQL module) NFDI4Culture: 1 Get GND IDs of Ferdinand Gregorovius's addressees", + "category": "Knowledge Graph", + "description": "This preset requires at least Facepager 4.6 \n\nThis preset accompanies our [Getting Started with Culture Knowledge Graph](https://github.com/strohne/Facepager/wiki/Getting-Started-with-CultureKnowledgeGraph) guide. \n\nIt retrives GND IDs of Ferdinand Gregorovius' addressees based on years as seed nodes. At the heart of the preset lies a SPARQL query. Making an effort to learn the basics of SPARQL's syntax will allow you to literally understand any such query and, more crucially, let's you write your own queries, thus empowering you to answer (research) questions of your interest. A great resources to start with is, for example, [Wikidata's introduction to SPARQL](https://www.wikidata.org/wiki/Wikidata:SPARQL_tutorial). For now, it will suffice to understand that the query returns the url, label, and date of any letter Gregorovius wrote during the specfied year as well as the GND IDs of all persons addressed or mention in each letter.\n\n**DISCLAIMER:** The Culture Knowledge Graph is work in progress. Currently, only a handful of datasets are accessible through its SPARQL endpoint using the NFDI4Culture Ontology. The User-Policy and guidelines of the Culture Knowledge Graph are being worked on as well. For the time being we advise you to be mindful of potential query limits.", + "module": "SPARQL", + "options": { + "auth": "disable", + "auth_prefix": "", + "auth_tokenname": "", + "auth_type": "Disable", + "auth_uri": "", + "basepath": "https://nfdi4culture.de/sparql", + "extension": "", + "format": "json", + "headers": { + "User-Agent": "FACEPAGERBOT/4.5 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.5" + }, + "key_paging": null, + "nodedata": "results.bindings", + "objectid": null, + "offset_start": 1, + "offset_step": 1, + "paging_stop": null, + "paging_type": null, + "param_paging": null, + "params": { + "format": "json", + "query": "PREFIX cto: \\\nPREFIX schema: \\\nPREFIX rdf: \\\nPREFIX rdfs: \\\nPREFIX n4c: \\\n\nSELECT ?letter ?letterLabel ?date ?gnd_id\nWHERE {\n n4c:E5378 schema:dataFeedElement/schema:item ?letter .\n ?letter rdfs:label ?letterLabel ;\n cto:creationDate ?date ;\n cto:relatedPerson ?gnd_url .\n FILTER(YEAR(?date) = ).\n BIND(REPLACE(STR(?gnd_url), \".*://.*/(.*?)\", \"$1\") AS ?gnd_id)\n} LIMIT 100" + }, + "redirect_uri": "", + "resource": "", + "token_uri": "" + }, + "speed": 200, + "saveheaders": false, + "timeout": 15, + "maxsize": 5, + "columns": [ + "date.value", + "gnd_id.value", + "letterLabel.value" + ] +} \ No newline at end of file diff --git a/presets/Knowledge_Graph-Wikidata_Get_gender_of_solo_artist_band_members_SPARQL_module_.fp4.json b/presets/Knowledge_Graph-Wikidata_Get_gender_of_solo_artist_band_members_SPARQL_module_.fp4.json new file mode 100644 index 00000000..29d468e0 --- /dev/null +++ b/presets/Knowledge_Graph-Wikidata_Get_gender_of_solo_artist_band_members_SPARQL_module_.fp4.json @@ -0,0 +1,41 @@ +{ + "name": "(SPARQL module) Wikidata: Get gender of solo artist/band members", + "category": "Knowledge Graph", + "description": "This preset requires at least Facepager 4.6 \n\nThis preset fetches a list of muscial artists and their sex or gender as provided by Wikidata. To start add the English name (label) of an arbitary musical solo artist or band as seed node, e.g. \"Peter Maffay\" or \"Coldplay\". The query automatically extracts names of band members and their respective on the basis of the group's name, e.g. \"Coldplay\". \n\nSee https://www.wikidata.org/ for further information. You can try out different queries on https://query.wikidata.org/ (paste the value of the query parameter).\n\nAlways ensure to comply to Wikidata's User-Agent policy and be mindful of potential query limits.\n\nPlease be aware that expressions in angle brackets are handled as placeholder by Facepager. Thus, the brackets of the prefixes have to be escaped with a backslash.\n\nPagination is handled by the placeholder.", + "module": "SPARQL", + "options": { + "basepath": "https://query.wikidata.org/sparql", + "extension": "", + "paging_type": null, + "key_paging": null, + "paging_stop": null, + "param_paging": null, + "offset_start": 1, + "offset_step": 1, + "nodedata": "results.bindings", + "objectid": null, + "auth_type": "Disable", + "auth_uri": "", + "redirect_uri": "", + "token_uri": "", + "auth": "disable", + "auth_prefix": "", + "auth_tokenname": "", + "resource": "", + "format": "json", + "headers": { + "User-Agent": "FACEPAGERBOT/4.5 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.5" + }, + "params": { + "query": "SELECT distinct ?entityLabel ?genderLabel WHERE {\n \n # Search for solo artist or band\n ?entity rdfs:label \"\"@en.\n \n # Check if ?entity is solo artist or band (musical group)\n {\n # If solo artist, get sex or gender\n ?entity wdt:P31 wd:Q5. \n ?entity wdt:P21 ?gender. \n }\n UNION\n {\n # If band (musical group), first get all members then their sex or gender \n ?entity wdt:P31 wd:Q215380. \n ?entity wdt:P527 ?member. \n ?member wdt:P21 ?gender. \n ?member rdfs:label ?entityLabel.\n FILTER(LANG(?entityLabel) = \"en\")\n }\n SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\". }\n}", + "format": "json" + } + }, + "speed": 200, + "saveheaders": false, + "timeout": 15, + "maxsize": 5, + "columns": [ + "genderLabel.value" + ] +} \ No newline at end of file diff --git a/presets/Knowledge_Graph-Wikidata_Get_list_of_biblical_characters_SPARQL_module_.fp4.json b/presets/Knowledge_Graph-Wikidata_Get_list_of_biblical_characters_SPARQL_module_.fp4.json new file mode 100644 index 00000000..e6f24a08 --- /dev/null +++ b/presets/Knowledge_Graph-Wikidata_Get_list_of_biblical_characters_SPARQL_module_.fp4.json @@ -0,0 +1,46 @@ +{ + "name": "(SPARQL module) Wikidata: Get list of biblical characters", + "category": "Knowledge Graph", + "description": "This preset requires at least Facepager 4.6. \n\nThis preset gets a list of biblical characters, along with the sections in which they occur, from WikiData. Add an arbitrary seed node and fetch data.\n\nSee https://www.wikidata.org/wiki/Wikidata:Lists/List_of_biblical_characters\n\nYou can try out different query on https://query.wikidata.org/ (paste the value of the query parameter to get started).\n\n\nPlease be aware that expressions in angle brackets are handled as placeholder by Facepager. Thus, if your query contains angle brackets, escape them with a backslash. Pagination is handled by the placeholder.", + "module": "SPARQL", + "options": { + "basepath": "https://query.wikidata.org/sparql", + "extension": "", + "paging_type": null, + "key_paging": null, + "paging_stop": null, + "param_paging": null, + "offset_start": 1, + "offset_step": 1, + "nodedata": "results.bindings", + "objectid": "figure.value", + "auth_type": "Disable", + "auth_uri": "", + "redirect_uri": "", + "token_uri": "", + "auth": "disable", + "auth_prefix": "", + "auth_tokenname": "", + "resource": "", + "format": "json", + "headers": { + "User-Agent": "FACEPAGERBOT/4.5 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.5" + }, + "params": { + "query": "SELECT DISTINCT ?figure ?figureLabel ?figureDescription ?section ?sectionLabel ?sectionDescription WHERE {\n\n ?figure wdt:P31 wd:Q12405827 .\n ?figure wdt:P1441 ?section .\n wd:Q1845 (wdt:P527)* ?chapter . \n wd:Q1845 (wdt:P527)* ?chapter .\n \n SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\".} \n}\nLIMIT 10", + "format": "json" + } + }, + "speed": 200, + "saveheaders": false, + "timeout": 15, + "maxsize": 5, + "columns": [ + "figureLabel.value", + "figureDescription.value", + "sectionLabel.value", + "sectionDescription.value", + "figure.value", + "section.value" + ] +} \ No newline at end of file diff --git a/presets/Knowledge_Graph-Wikidata_Which_politicans_did_not_die_on_natural_causes_SPARQL_module_.fp4.json b/presets/Knowledge_Graph-Wikidata_Which_politicans_did_not_die_on_natural_causes_SPARQL_module_.fp4.json new file mode 100644 index 00000000..9b7007b8 --- /dev/null +++ b/presets/Knowledge_Graph-Wikidata_Which_politicans_did_not_die_on_natural_causes_SPARQL_module_.fp4.json @@ -0,0 +1,44 @@ +{ + "name": "(SPARQL module) Wikidata: Which politicans did not die on natural causes?", + "category": "Knowledge Graph", + "description": "This preset requires at least Facepager 4.6. \n\nThis preset fetches a list of politicians from WikiData that did not die on natural causes. Add an arbitrary node as seed node, e.g. \"politicians\".\n\nSee https://www.wikidata.org/ for further information. You can try out different queries on https://query.wikidata.org/ (paste the value of the query parameter).\n\nPlease be aware that expressions in angle brackets are handled as placeholder by Facepager. Thus, the brackets of the prefixes have to be escaped with a backslash.\n\nPagination is handled by the placeholder.", + "module": "SPARQL", + "options": { + "auth": "disable", + "auth_prefix": "", + "auth_tokenname": "", + "auth_type": "Disable", + "auth_uri": "", + "basepath": "https://query.wikidata.org/sparql", + "extension": "", + "format": "json", + "headers": { + "User-Agent": "FACEPAGERBOT/4.5 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.5" + }, + "key_paging": null, + "nodedata": "results.bindings", + "objectid": "politician.value", + "offset_start": 1, + "offset_step": 1, + "paging_stop": null, + "paging_type": null, + "param_paging": null, + "params": { + "format": "json", + "query": "PREFIX wd: \\ \nPREFIX wdt: \\\n\nSELECT DISTINCT ?politician ?cause ?politicianLabel ?causeLabel WHERE {\n ?politician wdt:P106 wd:Q82955 . # find items that have \"occupation (P106): politician (Q82955)\"\n ?politician wdt:P509|wdt:P1196 ?cause . # with a P509 (cause of death) or P1196 (manner of death) claim\n ?cause wdt:P279* wd:Q149086 . # ... where the cause is a subclass of (P279*) homicide (Q149086)\n # ?politician wdt:P39 wd:Q11696 . # Uncomment this line to include only U.S. Presidents \n SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\" }\n} LIMIT 10" + }, + "redirect_uri": "", + "resource": "", + "token_uri": "" + }, + "speed": 200, + "saveheaders": false, + "timeout": 15, + "maxsize": 5, + "columns": [ + "politicianLabel.value", + "causeLabel.value", + "politician.value", + "cause.value" + ] +} \ No newline at end of file diff --git a/presets/Knowledge_Graph-Wikidata_Writers_Network_SPARQL_module_.fp4.json b/presets/Knowledge_Graph-Wikidata_Writers_Network_SPARQL_module_.fp4.json new file mode 100644 index 00000000..a5906e43 --- /dev/null +++ b/presets/Knowledge_Graph-Wikidata_Writers_Network_SPARQL_module_.fp4.json @@ -0,0 +1,43 @@ +{ + "name": "(SPARQL module) Wikidata: Writers Network", + "category": "Knowledge Graph", + "description": "This preset requires at least Facepager 4.6. \n\nThis preset accompanies the [Getting Started with Wikidata](https://github.com/strohne/Facepager/wiki/Getting-Started-with-Wikidata) tutorial. It fetches a list of German writers as well as cities relevant to them based on literary movements. To start add a Q-identifier of an arbitary literary movement as seed node, e.g. \"Q37068\" for romanticism.\n\nSee https://www.wikidata.org/ for further information. You can try out different queries on https://query.wikidata.org/ (paste the value of the query parameter).\n\nAlways ensure to comply to [Wikidata's User-Agent policy](https://foundation.wikimedia.org/wiki/Policy:User-Agent_policy) and be mindful of potential [query limits](https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Query_limits).\n\nPlease be aware that expressions in angle brackets are handled as placeholder by Facepager. Thus, the brackets of the prefixes have to be escaped with a backslash.", + "module": "SPARQL", + "options": { + "basepath": "https://query.wikidata.org/sparql", + "extension": "", + "paging_type": null, + "key_paging": null, + "paging_stop": null, + "param_paging": null, + "offset_start": 1, + "offset_step": 1, + "nodedata": "results.bindings", + "objectid": null, + "auth_type": "Disable", + "auth_uri": "", + "redirect_uri": "", + "token_uri": "", + "auth": "disable", + "auth_prefix": "", + "auth_tokenname": "", + "resource": "", + "format": "json", + "headers": { + "User-Agent": "FACEPAGERBOT/4.5 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.5" + }, + "params": { + "query": "SELECT DISTINCT ?writer ?writerLabel ?place ?placeLabel ?movement\nWHERE\n{\n # get all German writers of a literary movement\n ?writer wdt:P135 wd:;\n wdt:P106 wd:Q36180;\n wdt:P6886 wd:Q188.\n \n wd: rdfs:label ?movement.\n filter(lang(?movement) = \"en\")\n \n # relevant places\n ?writer wdt:P19|wdt:P20|(wdt:P69/wdt:P131)|(wdt:P108/wdt:P131) ?place.\n\n SERVICE wikibase:label { bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],en\". }\n \n}", + "format": "json" + } + }, + "speed": 200, + "saveheaders": false, + "timeout": 15, + "maxsize": 5, + "columns": [ + "writerLabel.value", + "placeLabel.value", + "movement.value" + ] +} \ No newline at end of file diff --git a/src/Facepager.py b/src/Facepager.py index 56e6c619..ab21bf31 100644 --- a/src/Facepager.py +++ b/src/Facepager.py @@ -23,6 +23,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from settings import * + import sys import argparse from datetime import datetime @@ -33,7 +35,6 @@ from PySide2.QtGui import * from PySide2.QtWidgets import QWidget, QStyleFactory, QMainWindow - from icons import * from widgets.datatree import DataTree from widgets.dictionarytree import DictionaryTree @@ -70,7 +71,7 @@ class MainWindow(QMainWindow): def __init__(self,central=None): super(MainWindow,self).__init__() - self.setWindowTitle("Facepager 4.5") + self.setWindowTitle("Facepager " + settings['version']) self.setWindowIcon(QIcon(":/icons/icon_facepager.png")) QApplication.setAttribute(Qt.AA_DisableWindowContextHelpButton) @@ -78,7 +79,7 @@ def __init__(self,central=None): # This is needed to display the app icon on the taskbar on Windows 7 if os.name == 'nt': import ctypes - myappid = 'Facepager.4.5' # arbitrary string + myappid = 'Facepager.' + settings['version'] # arbitrary string ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(myappid) self.setMinimumSize(1100,680) @@ -384,6 +385,7 @@ def createUI(self): self.RequestTabs.addTab(TwitterStreamingTab(self),"Twitter Streaming") self.RequestTabs.addTab(FacebookTab(self), "Facebook") self.RequestTabs.addTab(AmazonTab(self),"Amazon") + self.RequestTabs.addTab(SparqlTab(self),"SPARQL") self.RequestTabs.addTab(GenericTab(self),"Generic") module = self.settings.value('module',False) @@ -565,7 +567,7 @@ def writeSettings(self): self.settings.beginGroup("MainWindow") self.settings.setValue("size", self.size()) self.settings.setValue("pos", self.pos()) - self.settings.setValue("version","4.5") + self.settings.setValue("version",settings['version']) self.settings.endGroup() diff --git a/src/actions.py b/src/actions.py index cb6d921d..f4b166f1 100644 --- a/src/actions.py +++ b/src/actions.py @@ -356,12 +356,19 @@ def getDatabaseName(self): return (self.mainWindow.database.filename) def queryPipeline(self, pipeline, indexes=None): + """ + Query presets step by step + + :param pipeline: An object with the pipeline item in the 'item' key and the presets in the 'presets' key + :param indexes: + :return: + """ columns = [] - for preset in pipeline: + pipelineItem = pipeline['item'] + for preset in pipeline['presets']: # Select item in preset window - item = preset.get('item') - if item is not None: - self.mainWindow.presetWindow.presetList.setCurrentItem(item) + if pipelineItem is not None: + self.mainWindow.presetWindow.selectItem(pipelineItem, preset.get('step', 0)) columns.extend(preset.get('columns', [])) module = preset.get('module') @@ -377,8 +384,10 @@ def queryPipeline(self, pipeline, indexes=None): # Set columns columns = list(dict.fromkeys(columns)) - self.mainWindow.fieldList.setPlainText("\n".join(columns)) - self.showColumns() + if columns is not None: + self.mainWindow.fieldList.setPlainText("\n".join(columns)) + self.mainWindow.guiActions.showColumns() + @@ -1083,7 +1092,6 @@ def treeNodeSelected(self, current): if self.mainWindow.transferWindow.isVisible(): self.mainWindow.transferWindow.updateNode(current) - #select level level = 0 c = current diff --git a/src/apimodules.py b/src/apimodules.py index 285967b5..16e315e6 100644 --- a/src/apimodules.py +++ b/src/apimodules.py @@ -38,12 +38,14 @@ from server import LoginServer from widgets.paramedit import * from utilities import * +from settings import * try: from credentials import * except ImportError: credentials = {} + class ApiTab(QScrollArea): """ Generic API Tab Class @@ -65,7 +67,7 @@ def __init__(self, mainWindow=None, name="NoName"): self.connected = False self.lastrequest = None self.speed = None - self.appusage = 0 # Percent of rate limit + self.appusage = 0 # Percent of rate limit self.appusageLimit = 90 # Throttles speed down to 1 if exceeded self.lock_session = threading.Lock() self.sessions = [] @@ -75,8 +77,8 @@ def __init__(self, mainWindow=None, name="NoName"): self.mainLayout.setRowWrapPolicy(QFormLayout.DontWrapRows) self.mainLayout.setFormAlignment(Qt.AlignLeft | Qt.AlignTop) self.mainLayout.setLabelAlignment(Qt.AlignLeft) - self.mainLayout.setFieldGrowthPolicy(QFormLayout.AllNonFixedFieldsGrow) - self.mainLayout.setSizeConstraint(QLayout.SetMaximumSize) #QLayout.SetMinimumSize + self.mainLayout.setFieldGrowthPolicy(QFormLayout.AllNonFixedFieldsGrow) + self.mainLayout.setSizeConstraint(QLayout.SetMaximumSize) # QLayout.SetMinimumSize # Extra layout self.extraLayout = QFormLayout() @@ -96,16 +98,15 @@ def __init__(self, mainWindow=None, name="NoName"): page.setLayout(pagelayout) self.setWidget(page) self.setStyleSheet("QScrollArea {border:0px;background-color:transparent;}") - page.setAutoFillBackground(False) #import: place after setStyleSheet + page.setAutoFillBackground(False) # import: place after setStyleSheet self.setWidgetResizable(True) - # Popup window for auth settings self.authWidget = QWidget() # Default settings try: - self.defaults = credentials.get(name.lower().replace(' ','_'),{}) + self.defaults = credentials.get(name.lower().replace(' ', '_'), {}) except NameError: self.defaults = {} @@ -135,17 +136,17 @@ def parseURL(self, url): return path, query - def parsePlaceholders(self,pattern,nodedata,paramdata={},options = {}): + def parsePlaceholders(self, pattern, nodedata, paramdata={}, options={}): if not pattern: return pattern - elif isinstance(pattern,list): + elif isinstance(pattern, list): return [self.parsePlaceholders(x, nodedata, paramdata, options) for x in pattern] else: pattern = str(pattern) - #matches = re.findall(ur"<([^>]*>", pattern) - #matches = re.findall(ur"(?]*?)(?", pattern) - #Find placeholders in brackets, ignoring escaped brackets (escape character is backslash) + # matches = re.findall(ur"<([^>]*>", pattern) + # matches = re.findall(ur"(?]*?)(?", pattern) + # Find placeholders in brackets, ignoring escaped brackets (escape character is backslash) matches = re.findall(r"(?]*?(?", pattern) for match in matches: @@ -156,29 +157,29 @@ def parsePlaceholders(self,pattern,nodedata,paramdata={},options = {}): elif key == 'None': value = '' elif key == 'Object ID': - value = {'Object ID':str(nodedata['objectid'])} + value = {'Object ID': str(nodedata['objectid'])} name, value = extractValue(value, match, folder=options.get('folder', '')) else: - name, value = extractValue(nodedata['response'], match, folder=options.get('folder','')) + name, value = extractValue(nodedata['response'], match, folder=options.get('folder', '')) if (pattern == '<' + match + '>'): pattern = value return pattern - else: - #Mask special characters - value = value.replace('\\','\\\\') - value = value.replace('<','\\<') - value = value.replace('>','\\>') - + else: + # Mask special characters + value = value.replace('\\', '\\\\') + value = value.replace('<', '\\<') + value = value.replace('>', '\\>') + pattern = pattern.replace('<' + match + '>', value) - + pattern = pattern.replace('\\<', '<') pattern = pattern.replace('\\>', '>') pattern = pattern.replace('\\\\', '\\') return pattern - def getURL(self, urlpath, params, nodedata,options): + def getURL(self, urlpath, params, nodedata, options): """ Replaces the Facepager placeholders ("<",">") by the Object-ID or any other Facepager-Placeholder @@ -187,7 +188,8 @@ def getURL(self, urlpath, params, nodedata,options): urlpath, urlparams = self.parseURL(urlpath) # Filter empty params - params = {name: params[name] for name in params if (name != '') and (name != '') and (params[name] != '')} + params = {name: params[name] for name in params if + (name != '') and (name != '') and (params[name] != '')} # Collect template parameters (= placeholders) templateparams = {} @@ -204,7 +206,7 @@ def getURL(self, urlpath, params, nodedata,options): if not match: # Replace placeholders in parameter value value = self.parsePlaceholders(params[name], nodedata, templateparams, options) - if isinstance(value,list): + if isinstance(value, list): urlparams[name] = [str(x) for x in value] else: urlparams[name] = str(value) @@ -227,51 +229,62 @@ def getLogURL(self, urlpath, urlparams, options, removesecrets=True): return url - def getPayload(self,payload, params, nodedata,options, logProgress=None): - #Return nothing - if (payload is None) or (payload == ''): + def getPayload(self, payload, params, nodedata, options, logProgress=None): + """ + Encodes the payload + + :param payload: + :param params: + :param nodedata: + :param options: + :param logProgress: + :return: + """ + + # Return nothing + if (payload is None) or (payload == ''): return None - + # Parse JSON and replace placeholders in values - elif options.get('encoding','') == 'multipart/form-data': - #payload = json.loads(payload) + elif options.get('encoding', '') == 'multipart/form-data': + # payload = json.loads(payload) for name in payload: value = payload[name] - + try: value = json.loads(value) except: - pass - - # Files (convert dict to tuple) - if isinstance(value,dict): - filename = self.parsePlaceholders(value.get('name',''), nodedata, params,options) - filedata = self.parsePlaceholders(value.get('data',''), nodedata, params,options) - filetype = self.parsePlaceholders(value.get('type',''), nodedata, params,options) - payload[name] = (filename,filedata,filetype) - + pass + + # Files (convert dict to tuple) + if isinstance(value, dict): + filename = self.parsePlaceholders(value.get('name', ''), nodedata, params, options) + filedata = self.parsePlaceholders(value.get('data', ''), nodedata, params, options) + filetype = self.parsePlaceholders(value.get('type', ''), nodedata, params, options) + payload[name] = (filename, filedata, filetype) + # Strings else: value = payload[name] - payload[name] = self.parsePlaceholders(value, nodedata, params,options) + payload[name] = self.parsePlaceholders(value, nodedata, params, options) def callback(monitor): if logProgress is not None: logProgress({'current': monitor.bytes_read, 'total': monitor.len}) payload = MultipartEncoder(fields=payload) - payload = MultipartEncoderMonitor(payload,callback) + payload = MultipartEncoderMonitor(payload, callback) return payload - + # Replace placeholders in string and setup progress callback else: def callback(current, total): if logProgress is not None: logProgress({'current': current, 'total': total}) - payload = self.parsePlaceholders(payload, nodedata, params,options) - payload = BufferReader(payload,callback) + payload = self.parsePlaceholders(payload, nodedata, params, options) + payload = BufferReader(payload, callback) return payload # Gets data from input fields or defaults (never gets credentials from default values!) @@ -279,25 +292,33 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' options = {} defaults = self.getDefaultAndDocOptions() - #options['module'] = self.name + # options['module'] = self.name - #options for request + # Request options try: options['basepath'] = self.basepathEdit.currentText().strip() + except AttributeError: + pass + + try: options['resource'] = self.resourceEdit.currentText().strip() + except AttributeError: + pass + + try: options['params'] = self.paramEdit.getParams() except AttributeError: pass # Extension (for Twitter, deprecated) - options['extension'] = defaults.get('extension','') + options['extension'] = defaults.get('extension', '') if (options['extension'] != '') and options['resource'].endswith(options['extension']): options['extension'] = '' - #headers and verbs + # Headers and verbs try: options['headers'] = self.headerEdit.getParams() - options['verb'] = self.verbEdit.currentText().strip() + options['verb'] = self.verbEdit.currentText().strip() except AttributeError: pass # @@ -306,17 +327,17 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' # if doc_resource == '': # doc_resource = '0' - #format + # Format try: options['format'] = self.formatEdit.currentText().strip() except AttributeError: pass - #payload + # Payload try: - if options.get('verb','GET') in ['POST','PUT','PATCH']: + if options.get('verb', 'GET') in ['POST', 'PUT', 'PATCH']: options['encoding'] = self.encodingEdit.currentText().strip() - + if options['encoding'] == 'multipart/form-data': options['payload'] = self.multipartEdit.getParams() else: @@ -330,17 +351,23 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' except AttributeError: pass - #paging + # Paging try: options['pages'] = self.pagesEdit.value() except AttributeError: pass try: - options['paging_type'] = self.pagingTypeEdit.currentText().strip() if self.pagingTypeEdit.currentText() != "" else defaults.get('paging_type', '') - options['key_paging'] = self.pagingkeyEdit.text() if self.pagingkeyEdit.text() != "" else defaults.get('key_paging',None) - options['paging_stop'] = self.pagingstopEdit.text() if self.pagingstopEdit.text() != "" else defaults.get('paging_stop',None) - options['param_paging'] = self.pagingparamEdit.text() if self.pagingparamEdit.text() != "" else defaults.get('param_paging',None) + options[ + 'paging_type'] = self.pagingTypeEdit.currentText().strip() if self.pagingTypeEdit.currentText() != "" else defaults.get( + 'paging_type', '') + options['key_paging'] = self.pagingkeyEdit.text() if self.pagingkeyEdit.text() != "" else defaults.get( + 'key_paging', None) + options['paging_stop'] = self.pagingstopEdit.text() if self.pagingstopEdit.text() != "" else defaults.get( + 'paging_stop', None) + options[ + 'param_paging'] = self.pagingparamEdit.text() if self.pagingparamEdit.text() != "" else defaults.get( + 'param_paging', None) options['offset_start'] = self.offsetStartEdit.value() options['offset_step'] = self.offsetStepEdit.value() except AttributeError: @@ -367,10 +394,11 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' options.pop('key_paging') options.pop('paging_stop') - #options for data handling + # Data handling options try: options['nodedata'] = self.extractEdit.text() if self.extractEdit.text() != "" else defaults.get('nodedata') - options['objectid'] = self.objectidEdit.text() if self.objectidEdit.text() != "" else defaults.get('objectid') + options['objectid'] = self.objectidEdit.text() if self.objectidEdit.text() != "" else defaults.get( + 'objectid') except AttributeError: options['nodedata'] = defaults.get('nodedata') options['objectid'] = defaults.get('objectid') @@ -386,11 +414,13 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' except AttributeError: pass - # Options not saved to preset but to settings if purpose != 'preset': # query type - options['querytype'] = self.name + ':' + self.resourceEdit.currentText() + try: + options['querytype'] = self.name + ':' + self.resourceEdit.currentText() + except AttributeError: + options['querytype'] = self.name # uploadfolder try: @@ -407,17 +437,17 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' try: options['access_token'] = self.tokenEdit.text() except AttributeError: - pass + pass try: options['access_token_secret'] = self.tokensecretEdit.text() except AttributeError: - pass - + pass + try: options['client_id'] = self.clientIdEdit.text() except AttributeError: - pass + pass try: options['client_secret'] = self.clientSecretEdit.text() @@ -427,66 +457,77 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' return options def updateBasePath(self, options=None): - if options is None: - basepath = self.basepathEdit.currentText().strip() - options = {'basepath' : basepath} - else: - basepath = options.get('basepath', '') - self.basepathEdit.setEditText(basepath) + try: + if options is None: + basepath = self.basepathEdit.currentText().strip() + options = {'basepath': basepath} + else: + basepath = options.get('basepath', '') + self.basepathEdit.setEditText(basepath) - index = self.basepathEdit.findText(basepath) - if index != -1: - self.basepathEdit.setCurrentIndex(index) + index = self.basepathEdit.findText(basepath) + if index != -1: + self.basepathEdit.setCurrentIndex(index) - # Get general doc - apidoc = self.mainWindow.apiWindow.getApiDoc(self.name, basepath) + # Get general doc + apidoc = self.mainWindow.apiWindow.getApiDoc(self.name, basepath) - # apidoc = self.basepathEdit.itemData(index, Qt.UserRole) + # apidoc = self.basepathEdit.itemData(index, Qt.UserRole) - # Add endpoints in reverse order - self.resourceEdit.clear() - if apidoc and isinstance(apidoc, dict): - endpoints = apidoc.get("paths", {}) - paths = endpoints.keys() - for path in list(paths): - operations = endpoints[path] - path = path.replace("{", "<").replace("}", ">") + # Add endpoints in reverse order + self.resourceEdit.clear() + if apidoc and isinstance(apidoc, dict): + endpoints = apidoc.get("paths", {}) + paths = endpoints.keys() + for path in list(paths): + operations = endpoints[path] + path = path.replace("{", "<").replace("}", ">") - self.resourceEdit.addItem(path) - idx = self.resourceEdit.count() - 1 - self.resourceEdit.setItemData(idx, wraptip(getDictValue(operations, "get.summary", "")), Qt.ToolTipRole) + self.resourceEdit.addItem(path) + idx = self.resourceEdit.count() - 1 + self.resourceEdit.setItemData(idx, wraptip(getDictValue(operations, "get.summary", "")), + Qt.ToolTipRole) - # store params for later use in onChangedResource - self.resourceEdit.setItemData(idx, operations, Qt.UserRole) + # store params for later use in onChangedResource + self.resourceEdit.setItemData(idx, operations, Qt.UserRole) - self.buttonApiHelp.setVisible(True) - else: - self.resourceEdit.insertItem(0, "/") + self.buttonApiHelp.setVisible(True) + else: + self.resourceEdit.insertItem(0, "/") + except AttributeError: + # In case one of the inputs does not exist + pass def updateResource(self, options=None): - if options is None: - resource = self.resourceEdit.currentText().strip() - options = {'resource' : resource} - else: - resource = options.get('resource', '') - self.resourceEdit.setEditText(resource) + try: + if options is None: + resource = self.resourceEdit.currentText().strip() + else: + resource = options.get('resource', '') + self.resourceEdit.setEditText(resource) - index = self.resourceEdit.findText(resource) - if index != -1: - self.resourceEdit.setCurrentIndex(index) + index = self.resourceEdit.findText(resource) + if index != -1: + self.resourceEdit.setCurrentIndex(index) + operations = self.resourceEdit.itemData(index, Qt.UserRole) + params = getDictValue(operations, "get.parameters", False) if operations else [] - operations = self.resourceEdit.itemData(index, Qt.UserRole) - params = getDictValue(operations, "get.parameters", False) if operations else [] + # Set param names + self.paramEdit.setNameOptionsAll(params) - # Set param names - self.paramEdit.setNameOptionsAll(params) + except AttributeError: + # In case one of the inputs does not exist + pass - # Populates input fields from loaded options and presets - # Select boxes are updated by onChangedBasepath and onChangedResource - # based on the API docs. - # @settings Dict with options - def setSettings(self, settings = {}): + def setSettings(self, settings={}): + """ + Populates input fields from loaded options and presets + Select boxes are updated by onChangedBasepath and onChangedResource + based on the API docs. + :param settings: Dict with options + :return: + """ # Base path options = self.getDefaultAndDocOptions(settings) @@ -504,7 +545,11 @@ def setSettings(self, settings = {}): return options def updateParams(self, options): - self.paramEdit.setParams(options.get('params', '')) + try: + self.paramEdit.setParams(options.get('params', '')) + except AttributeError: + # In case one of the inputs does not exist + pass def updateOptions(self, options): @@ -618,7 +663,7 @@ def loadSettings(self): self.setSettings(options) @Slot(str) - def logMessage(self,message): + def logMessage(self, message): self.mainWindow.logmessage(message) def reloadDoc(self): @@ -634,7 +679,7 @@ def loadDoc(self): # Add base paths self.basepathEdit.clear() urls = self.mainWindow.apiWindow.getApiBasePaths(self.name) - self.basepathEdit.insertItems(0,urls) + self.basepathEdit.insertItems(0, urls) # TODO: set API Docs as item data @@ -646,7 +691,7 @@ def showDoc(self): path = self.resourceEdit.currentText().strip() self.mainWindow.apiWindow.showDoc(self.name, basepath, path) - def getDefaultAndDocOptions(self, options = {}): + def getDefaultAndDocOptions(self, options={}): # Set default options defaults = self.defaults.copy() defaults.update(self.getDocOptions()) @@ -665,13 +710,16 @@ def getDocOptions(self): # Get general doc basepath = self.basepathEdit.currentText().strip() - apidoc = self.mainWindow.apiWindow.getApiDoc(self.name,basepath) + apidoc = self.mainWindow.apiWindow.getApiDoc(self.name, basepath) - # Get response doc - resourceidx = self.resourceEdit.findText(self.resourceEdit.currentText()) - operations = self.resourceEdit.itemData(resourceidx,Qt.UserRole) if resourceidx != -1 else {} - schema = getDictValue(operations, "get.responses.200.content.application/json.schema", []) if operations else [] + # Get response doc from the resourceEdit if a resourceEdit exists + try: + resourceIdx = self.resourceEdit.findText(self.resourceEdit.currentText()) + operations = self.resourceEdit.itemData(resourceIdx, Qt.UserRole) if resourceIdx != -1 else {} + except AttributeError: + operations = {} + schema = getDictValue(operations, "get.responses.200.content.application/json.schema", []) if operations else [] options = {} # Params @@ -711,7 +759,6 @@ def getDocOptions(self): options['auth'] = getDictValueOrNone(authorization, 'auth_method') options['auth_tokenname'] = getDictValueOrNone(authorization, 'token_name') - # Extract options from response reference if 'x-facepager-extract' in schema: options['nodedata'] = schema.get('x-facepager-extract') @@ -719,31 +766,41 @@ def getDocOptions(self): if 'x-facepager-objectid' in schema: options['objectid'] = schema.get('x-facepager-objectid') - options = {k: v for k, v in options.items() if v is not None} return options - def initInputs(self): + def initBasicInputs(self): ''' Create base path edit, resource edit and param edit - Set resource according to the APIdocs, if any docs are available ''' - #Base path + # Base path + self.initBasepathInput() + self.initResourceInput() + self.initParamsInput() + + def initBasepathInput(self, label="Base path"): + ''' + Create base path edit + ''' self.basepathEdit = QComboBox(self) - if not self.defaults.get('basepath',None) is None: - self.basepathEdit.insertItems(0, [self.defaults.get('basepath','')]) + if not self.defaults.get('basepath', None) is None: + self.basepathEdit.insertItems(0, [self.defaults.get('basepath', '')]) self.basepathEdit.setEditable(True) - self.mainLayout.addRow("Base path", self.basepathEdit) + self.mainLayout.addRow(label, self.basepathEdit) self.basepathEdit.currentIndexChanged.connect(self.onChangedBasepath) - #Resource + def initResourceInput(self): + ''' + Create resource edit. + Set resource according to the APIdocs, if any docs are available + ''' self.resourceLayout = QHBoxLayout() - self.actionApiHelp = QAction('Open documentation if available.',self) + self.actionApiHelp = QAction('Open documentation if available.', self) self.actionApiHelp.setText('?') self.actionApiHelp.triggered.connect(self.showDoc) - self.buttonApiHelp =QToolButton(self) + self.buttonApiHelp = QToolButton(self) self.buttonApiHelp.setToolButtonStyle(Qt.ToolButtonTextOnly) self.buttonApiHelp.setDefaultAction(self.actionApiHelp) self.buttonApiHelp.setVisible(False) @@ -756,12 +813,16 @@ def initInputs(self): self.mainLayout.addRow("Resource", self.resourceLayout) - #Parameters + self.resourceEdit.currentIndexChanged.connect(self.onChangedResource) + + def initParamsInput(self): + ''' + Create parameter inputs + ''' self.paramEdit = QParamEdit(self) self.mainLayout.addRow("Parameters", self.paramEdit) - self.resourceEdit.currentIndexChanged.connect(self.onChangedResource) - def getFileFolderName(self,options, nodedata): + def getFileFolderName(self, options, nodedata): # Folder foldername = options.get('downloadfolder', None) if foldername == '': @@ -769,10 +830,10 @@ def getFileFolderName(self,options, nodedata): # File filename = options.get('filename', None) - if (filename is not None) and (filename == ''): + if (filename is not None) and (filename == ''): filename = None else: - filename = self.parsePlaceholders(filename,nodedata) + filename = self.parsePlaceholders(filename, nodedata) if filename == '': filename = None @@ -783,15 +844,15 @@ def getFileFolderName(self,options, nodedata): if fileext is not None and fileext == '': fileext = None elif fileext is not None and fileext != '': - fileext = self.parsePlaceholders(fileext,nodedata) + fileext = self.parsePlaceholders(fileext, nodedata) - return (foldername,filename,fileext) + return (foldername, filename, fileext) # Upload folder def initUploadFolderInput(self): self.folderwidget = QWidget() folderlayout = QHBoxLayout() - folderlayout.setContentsMargins(0,0,0,0) + folderlayout.setContentsMargins(0, 0, 0, 0) self.folderwidget.setLayout(folderlayout) self.folderEdit = QLineEdit() @@ -808,40 +869,41 @@ def initFileInputs(self): self.downloadfolderwidget = QWidget() folderlayout = QHBoxLayout() - folderlayout.setContentsMargins(0,0,0,0) + folderlayout.setContentsMargins(0, 0, 0, 0) self.downloadfolderwidget.setLayout(folderlayout) # Folder edit self.downloadfolderEdit = QLineEdit() self.downloadfolderEdit.setToolTip(wraptip("Select a folder if you want to save the responses to files.")) - folderlayout.addWidget(self.downloadfolderEdit,2) + folderlayout.addWidget(self.downloadfolderEdit, 2) # Select folder button - self.actionDownloadFolder = QAction('...',self) + self.actionDownloadFolder = QAction('...', self) self.actionDownloadFolder.setText('..') self.actionDownloadFolder.triggered.connect(self.selectDownloadFolder) - self.downloadfolderButton =QToolButton(self) + self.downloadfolderButton = QToolButton(self) self.downloadfolderButton.setToolButtonStyle(Qt.ToolButtonTextOnly) self.downloadfolderButton.setDefaultAction(self.actionDownloadFolder) - folderlayout.addWidget(self.downloadfolderButton,0) + folderlayout.addWidget(self.downloadfolderButton, 0) # filename - folderlayout.addWidget(QLabel("Filename"),0) + folderlayout.addWidget(QLabel("Filename"), 0) self.filenameEdit = QComboBox(self) - self.filenameEdit .setToolTip(wraptip("Set the filename, if you want to save the responses to files. usually is a good choice.")) - self.filenameEdit.insertItems(0, ['','']) + self.filenameEdit.setToolTip(wraptip( + "Set the filename, if you want to save the responses to files. usually is a good choice.")) + self.filenameEdit.insertItems(0, ['', '']) self.filenameEdit.setEditable(True) - folderlayout.addWidget(self.filenameEdit,1) - + folderlayout.addWidget(self.filenameEdit, 1) # fileext - folderlayout.addWidget(QLabel("Custom file extension"),0) + folderlayout.addWidget(QLabel("Custom file extension"), 0) self.fileextEdit = QComboBox(self) - self.fileextEdit .setToolTip(wraptip("Set the extension of the files, for example .json, .txt or .html. Set to to automatically guess from the response.")) - self.fileextEdit.insertItems(0, ['','.html','.txt']) + self.fileextEdit.setToolTip(wraptip( + "Set the extension of the files, for example .json, .txt or .html. Set to to automatically guess from the response.")) + self.fileextEdit.insertItems(0, ['', '.html', '.txt']) self.fileextEdit.setEditable(True) - folderlayout.addWidget(self.fileextEdit,1) - #layout.setStretch(2, 1) + folderlayout.addWidget(self.fileextEdit, 1) + # layout.setStretch(2, 1) self.extraLayout.addRow("Download", self.downloadfolderwidget) @@ -862,10 +924,9 @@ def pagingChanged(self): if self.pagingTypeEdit.count() < 2: self.pagingTypeEdit.hide() + def initPagingInputs(self, keys=False): + layout = QHBoxLayout() - def initPagingInputs(self,keys = False): - layout= QHBoxLayout() - if keys: # Paging type @@ -876,7 +937,8 @@ def initPagingInputs(self,keys = False): self.pagingTypeEdit.addItem('url') self.pagingTypeEdit.addItem('decrease') - self.pagingTypeEdit.setToolTip(wraptip("Select 'key' if the response contains data about the next page, e.g. page number or offset. Select 'count' if you want to increase the paging param by a fixed amount. Select 'url' if the response contains a complete URL to the next page.")) + self.pagingTypeEdit.setToolTip(wraptip( + "Select 'key' if the response contains data about the next page, e.g. page number or offset. Select 'count' if you want to increase the paging param by a fixed amount. Select 'url' if the response contains a complete URL to the next page.")) self.pagingTypeEdit.currentIndexChanged.connect(self.pagingChanged) layout.addWidget(self.pagingTypeEdit) layout.setStretch(0, 0) @@ -884,14 +946,15 @@ def initPagingInputs(self,keys = False): # Paging param self.pagingParamWidget = QWidget() self.pagingParamLayout = QHBoxLayout() - self.pagingParamLayout .setContentsMargins(0, 0, 0, 0) + self.pagingParamLayout.setContentsMargins(0, 0, 0, 0) self.pagingParamWidget.setLayout(self.pagingParamLayout) self.pagingParamLayout.addWidget(QLabel("Param")) self.pagingparamEdit = QLineEdit(self) - self.pagingparamEdit.setToolTip(wraptip("This parameter will be added to the query if you select key-pagination. The value is extracted by the paging key.")) + self.pagingparamEdit.setToolTip(wraptip( + "This parameter will be added to the query if you select key-pagination. The value is extracted by the paging key.")) self.pagingParamLayout.addWidget(self.pagingparamEdit) - self.pagingParamLayout.setStretch(0,0) + self.pagingParamLayout.setStretch(0, 0) self.pagingParamLayout.setStretch(1, 0) layout.addWidget(self.pagingParamWidget) @@ -900,12 +963,13 @@ def initPagingInputs(self,keys = False): # Paging key self.pagingKeyWidget = QWidget() self.pagingKeyLayout = QHBoxLayout() - self.pagingKeyLayout .setContentsMargins(0, 0, 0, 0) + self.pagingKeyLayout.setContentsMargins(0, 0, 0, 0) self.pagingKeyWidget.setLayout(self.pagingKeyLayout) self.pagingKeyLayout.addWidget(QLabel("Paging key")) self.pagingkeyEdit = QLineEdit(self) - self.pagingkeyEdit.setToolTip(wraptip("If the respsonse contains data about the next page, specify the key. The value will be added as paging parameter or used as the URL.")) + self.pagingkeyEdit.setToolTip(wraptip( + "If the respsonse contains data about the next page, specify the key. The value will be added as paging parameter or used as the URL.")) self.pagingKeyLayout.addWidget(self.pagingkeyEdit) self.pagingKeyLayout.setStretch(0, 0) self.pagingKeyLayout.setStretch(1, 1) @@ -942,12 +1006,13 @@ def initPagingInputs(self,keys = False): # Stop if layout.addWidget(QLabel("Stop key")) self.pagingstopEdit = QLineEdit(self) - self.pagingstopEdit.setToolTip(wraptip("Stops fetching data as soon as the given key is present but empty or false. For example, stops fetching if the value of 'hasNext' ist false, none or an empty list. Usually you can leave the field blank, since fetching will stop anyway when the paging key is empty.")) + self.pagingstopEdit.setToolTip(wraptip( + "Stops fetching data as soon as the given key is present but empty or false. For example, stops fetching if the value of 'hasNext' ist false, none or an empty list. Usually you can leave the field blank, since fetching will stop anyway when the paging key is empty.")) layout.addWidget(self.pagingstopEdit) layout.setStretch(4, 0) layout.setStretch(5, 1) - #Page count + # Page count layout.addWidget(QLabel("Maximum pages")) self.pagesEdit = QSpinBox(self) self.pagesEdit.setMinimum(1) @@ -960,12 +1025,12 @@ def initPagingInputs(self,keys = False): rowcaption = "Paging" else: - #Page count + # Page count self.pagesEdit = QSpinBox(self) self.pagesEdit.setMinimum(1) self.pagesEdit.setMaximum(50000) - layout.addWidget(self.pagesEdit) - + layout.addWidget(self.pagesEdit) + rowcaption = "Maximum pages" self.extraLayout.addRow(rowcaption, layout) @@ -974,128 +1039,126 @@ def initHeaderInputs(self): self.headerEdit = QParamEdit(self) self.mainLayout.addRow("Headers", self.headerEdit) - def initVerbInputs(self): # Verb and encoding self.verbEdit = QComboBox(self) - self.verbEdit.addItems(['GET','HEAD','POST','PUT','PATCH','DELETE']) + self.verbEdit.addItems(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE']) self.verbEdit.currentIndexChanged.connect(self.verbChanged) self.encodingLabel = QLabel("Encoding") self.encodingEdit = QComboBox(self) - self.encodingEdit.addItems(['','multipart/form-data']) + self.encodingEdit.addItems(['', 'multipart/form-data']) self.encodingEdit.currentIndexChanged.connect(self.verbChanged) - - layout= QHBoxLayout() + layout = QHBoxLayout() layout.addWidget(self.verbEdit) layout.setStretch(0, 1) layout.addWidget(self.encodingLabel) layout.addWidget(self.encodingEdit) layout.setStretch(2, 1) self.mainLayout.addRow("Method", layout) - + # Payload self.payloadWidget = QWidget() self.payloadLayout = QHBoxLayout() - self.payloadLayout.setContentsMargins(0,0,0,0) + self.payloadLayout.setContentsMargins(0, 0, 0, 0) self.payloadWidget.setLayout(self.payloadLayout) - + self.payloadEdit = QPlainTextEdit() self.payloadEdit.setLineWrapMode(QPlainTextEdit.NoWrap) self.payloadLayout.addWidget(self.payloadEdit) self.payloadLayout.setStretch(0, 1); - + self.multipartEdit = QParamEdit() self.payloadLayout.addWidget(self.multipartEdit) - self.payloadLayout.setStretch(0, 1); + self.payloadLayout.setStretch(2, 1); - self.payloadLayout.setStretch(2, 1); self.mainLayout.addRow("Payload", self.payloadWidget) def verbChanged(self): - if self.verbEdit.currentText() in ['GET','DELETE','HEAD']: + if self.verbEdit.currentText() in ['GET', 'DELETE', 'HEAD']: self.payloadWidget.hide() self.mainLayout.labelForField(self.payloadWidget).hide() - + self.encodingEdit.hide() - self.encodingLabel.hide() + self.encodingLabel.hide() self.folderwidget.hide() self.mainLayout.labelForField(self.folderwidget).hide() else: + self.payloadWidget.show() self.mainLayout.labelForField(self.payloadWidget).show() - #Encoding + # Encoding self.encodingEdit.show() self.encodingLabel.show() - - #Multipart + + # Multipart if self.encodingEdit.currentText().strip() == 'multipart/form-data': self.multipartEdit.show() self.payloadEdit.hide() - - #self.payloadEdit.setPlainText(json.dumps(self.multipartEdit.getParams(),indent=4,)) + + # self.payloadEdit.setPlainText(json.dumps(self.multipartEdit.getParams(),indent=4,)) else: self.payloadEdit.show() self.multipartEdit.hide() - - #Folder - self.folderwidget.show() - self.mainLayout.labelForField(self.folderwidget).show() - def initResponseInputs(self, format=False): - layout= QHBoxLayout() + # Folder + self.folderwidget.show() + self.mainLayout.labelForField(self.folderwidget).show() - if not format: - #Extract - self.extractEdit = QLineEdit(self) - layout.addWidget(self.extractEdit) - layout.setStretch(0, 1) + def initResponseInputs(self, format=True): + """ + Create inputs for extraction keys - layout.addWidget(QLabel("Key for Object ID")) - self.objectidEdit = QLineEdit(self) - layout.addWidget(self.objectidEdit) - layout.setStretch(2, 1) + :param format: Whether to create a format selector + :return: + """ + layout = QHBoxLayout() - #Add layout - self.extraLayout.addRow("Key to extract", layout) - else: - # Format + # Format + if format: self.formatEdit = QComboBox(self) - self.formatEdit.addItems(['json', 'text', 'links','xml','file']) + self.formatEdit.addItems(['json', 'text', 'links', 'xml', 'file', 'rdf', 'ttl', 'json-ld']) self.formatEdit.setToolTip("

JSON: default option, data will be parsed as JSON.

\

Text: data will not be parsed and embedded in JSON.

\

Links: data will be parsed as xml and links will be extracted (set key to extract to 'links' and key for Object ID to 'url').

\

XML: data will be parsed as XML and converted to JSON.

\

File: data will only be downloaded to files, specify download folder and filename.

") layout.addWidget(self.formatEdit) - layout.setStretch(0, 0) # self.formatEdit.currentIndexChanged.connect(self.formatChanged) - # Extract - layout.addWidget(QLabel("Key to extract")) - self.extractEdit = QLineEdit(self) - self.extractEdit.setToolTip(wraptip( - "If your data contains a list of objects, set the key of the list. Every list element will be adeded as a single node. Remaining data will be added as offcut node.")) - layout.addWidget(self.extractEdit) + layout.setStretch(0, 0) layout.setStretch(1, 0) layout.setStretch(2, 2) - - layout.addWidget(QLabel("Key for Object ID")) - self.objectidEdit = QLineEdit(self) - self.objectidEdit.setToolTip( - wraptip("If your data contains unique IDs for every node, define the corresponding key.")) - layout.addWidget(self.objectidEdit) layout.setStretch(3, 0) layout.setStretch(4, 2) - # Add layout - self.extraLayout.addRow("Response", layout) + rowLabel = "Response" + layout.addWidget(QLabel("Key to extract")) + else: + layout.setStretch(0, 1) + layout.setStretch(2, 1) + rowLabel = "Key to extract" + + # Extract + self.extractEdit = QLineEdit(self) + self.extractEdit.setToolTip(wraptip( + "If your data contains a list of objects, set the key of the list. Every list element will be adeded as a single node. Remaining data will be added as offcut node.")) + layout.addWidget(self.extractEdit) + + layout.addWidget(QLabel("Key for Object ID")) + self.objectidEdit = QLineEdit(self) + self.objectidEdit.setToolTip( + wraptip("If your data contains unique IDs for every node, define the corresponding key.")) + layout.addWidget(self.objectidEdit) + + # Add layout + self.extraLayout.addRow(rowLabel, layout) @Slot() - def onChangedBasepath(self, index = None): + def onChangedBasepath(self, index=None): ''' Handles the automated resource suggestion for the selected API based on the OpenAPI specification 3.0.0 @@ -1113,7 +1176,7 @@ def onChangedBasepath(self, index = None): self.updateOptions(defaults) @Slot() - def onChangedResource(self, index = None): + def onChangedResource(self, index=None): ''' Handles the automated parameter suggestion for the selected API endpoint based on the OpenAPI specification 3.0.0 @@ -1130,9 +1193,8 @@ def onChangedResource(self, index = None): self.updateParams(defaults) self.updateOptions(defaults) - @Slot() - def onChangedParam(self,index=0): + def onChangedParam(self, index=0): pass def getProxies(self, reload=False): @@ -1151,7 +1213,7 @@ def getProxies(self, reload=False): self.proxies = self.proxies[1:] + self.proxies[:1] if proxy.startswith('http'): - proxy_http = "http://"+re.sub('^https?://', '', proxy) + proxy_http = "http://" + re.sub('^https?://', '', proxy) proxy_https = "https://" + re.sub('^https?://', '', proxy) return {'http': proxy_http, 'https': proxy_https} elif proxy != "": @@ -1187,27 +1249,28 @@ def initPagingOptions(self, data, options): options['params'][options.get('param_paging', '')] = offset # paging by key (continue previous fetching process based on last fetched child offcut node) - elif (options.get('paging_type') == "key") and (options.get('key_paging') is not None) and (options.get('param_paging') is not None): + elif (options.get('paging_type') == "key") and (options.get('key_paging') is not None) and ( + options.get('param_paging') is not None): # Get cursor of last offcut node offcut = getDictValueOrNone(options, 'lastdata.response', dump=False) - cursor = getDictValueOrNone(offcut,options.get('key_paging')) - stopvalue = not extractValue(offcut,options.get('paging_stop'), dump = False, default = True)[1] + cursor = getDictValueOrNone(offcut, options.get('key_paging')) + stopvalue = not extractValue(offcut, options.get('paging_stop'), dump=False, default=True)[1] # Dont't fetch if already finished (=offcut without next cursor) - if options.get('resume',False) and (offcut is not None) and ((cursor is None) or stopvalue): + if options.get('resume', False) and (offcut is not None) and ((cursor is None) or stopvalue): return None # Continue / start fetching - elif (cursor is not None) : + elif (cursor is not None): options['params'][options['param_paging']] = cursor # url based paging elif (options.get('paging_type') == "url") and (options.get('key_paging') is not None): offcut = getDictValueOrNone(options, 'lastdata.response', dump=False) - url = getDictValueOrNone(offcut,options.get('key_paging')) + url = getDictValueOrNone(offcut, options.get('key_paging')) # Dont't fetch if already finished (=offcut without next cursor) - if options.get('resume',False) and (offcut is not None) and (url is None): + if options.get('resume', False) and (offcut is not None) and (url is None): return None if url is not None: @@ -1216,7 +1279,7 @@ def initPagingOptions(self, data, options): options['url'] = url elif (options.get('paging_type') == "decrease"): - node= getDictValueOrNone(options, 'lastdata.response', dump=False) + node = getDictValueOrNone(options, 'lastdata.response', dump=False) cursor = getDictValueOrNone(node, options.get('key_paging')) if (node is not None): @@ -1230,7 +1293,7 @@ def initPagingOptions(self, data, options): return None # break if "continue pagination" is checked and data already present - elif options.get('resume',False): + elif options.get('resume', False): offcut = getDictValueOrNone(options, 'lastdata.response', dump=False) # Dont't fetch if already finished (=offcut) @@ -1312,8 +1375,8 @@ def buildUrl(self, nodedata, options, logProgress=None): requestheaders = options.get('headers', {}) # Authorization - if options.get('auth','disable') != 'disable': - token = options.get('auth_prefix','') + options.get('access_token','') + if options.get('auth', 'disable') != 'disable': + token = options.get('auth_prefix', '') + options.get('access_token', '') if options.get('auth') == 'param': urlparams[options.get('auth_tokenname')] = token elif (options.get('auth') == 'header'): @@ -1373,13 +1436,13 @@ def closeSession(self, no=0): self.sessions[no].close() self.sessions[no] = None - def request(self, session_no=0, path=None, args=None, headers=None, method="GET", payload=None,foldername=None, - filename=None, fileext=None, format='json'): + def request(self, session_no=0, path=None, args=None, headers=None, method="GET", payload=None, foldername=None, + filename=None, fileext=None, format='json'): """ Start a new threadsafe session and request """ - def download(response,foldername=None,filename=None,fileext=None): + def download(response, foldername=None, filename=None, fileext=None): if foldername is not None and filename is not None: if fileext is None: contentype = response.headers.get("content-type") @@ -1389,8 +1452,7 @@ def download(response,foldername=None,filename=None,fileext=None): else: fileext = None - - fullfilename = makefilename(path,foldername, filename, fileext) + fullfilename = makefilename(path, foldername, filename, fileext) file = open(fullfilename, 'wb') else: fullfilename = None @@ -1408,7 +1470,6 @@ def download(response,foldername=None,filename=None,fileext=None): if file is not None: file.write(chunk) - out = content.getvalue() encoding = response.encoding encoding = cchardet.detect(out)['encoding'] if encoding is None else encoding @@ -1423,17 +1484,9 @@ def download(response,foldername=None,filename=None,fileext=None): if file is not None: file.close() - # if file is not None: - # try: - # for chunk in response.iter_content(1024): - # file.write(chunk) - # finally: - # file.close() - - return (fullfilename, out) - #Throttle speed + # Throttle speed if (self.speed is not None) and (self.lastrequest is not None): currentSpeed = 1 if self.appusage > self.appusageLimit else self.speed pause = ((60 * 1000) / float(currentSpeed)) - self.lastrequest.msecsTo(QDateTime.currentDateTime()) @@ -1464,8 +1517,9 @@ def download(response,foldername=None,filename=None,fileext=None): cookies = dict(item.split("=", maxsplit=1) for item in cookies.split(";")) # Send request - response = session.request(method,path, params=args, headers=headers, cookies=cookies, - data=payload, timeout=self.timeout,stream=True,verify=True) # verify=False + response = session.request(method, path, params=args, headers=headers, cookies=cookies, + data=payload, timeout=self.timeout, stream=True, + verify=True) # verify=False except (HTTPError, ConnectionError) as e: maxretries -= 1 @@ -1480,7 +1534,7 @@ def download(response,foldername=None,filename=None,fileext=None): else: break - if int(response.headers.get('content-length',0)) > (self.maxsize * 1024 * 1024): + if int(response.headers.get('content-length', 0)) > (self.maxsize * 1024 * 1024): raise DataTooBigError(f"File is too big, content length is {response.headers['content-length']}.") status = 'fetched' if response.ok else 'error' @@ -1489,8 +1543,8 @@ def download(response,foldername=None,filename=None,fileext=None): # Download data data = { - 'content-type': response.headers.get("content-type",""), - 'sourcepath': path,'sourcequery': args,'finalurl': response.url + 'content-type': response.headers.get("content-type", ""), + 'sourcepath': path, 'sourcequery': args, 'finalurl': response.url } fullfilename, content = download(response, foldername, filename, fileext) @@ -1499,62 +1553,18 @@ def download(response,foldername=None,filename=None,fileext=None): data['filename'] = os.path.basename(fullfilename) data['filepath'] = fullfilename - # Text - if format == 'text': - data['text'] = content # str(response.text) - - # Scrape links - elif format == 'links': - try: - links, base = extractLinks(content, response.url) - data['links'] = links - data['base'] = base - except Exception as e: - data['error'] = 'Could not extract Links.' - data['message'] = str(e) - data['response'] = content - - # JSON - elif format == 'json': - try: - data = json.loads(content) if content != '' else [] - except Exception as e: - # self.logMessage("No valid JSON data, try to convert XML to JSON ("+str(e)+")") - # try: - # data = xmlToJson(response.text) - # except: - data = { - 'error': 'Data could not be converted to JSON', - 'response': content, - 'exception':str(e) - } - - # JSON - elif format == 'xml': - try: - data = xmlToJson(content) - except Exception as e: - data = { - 'error': 'Data could not be converted to JSON', - 'response': content, - 'exception':str(e) - } - + data = self.postProcessData(data, content, response, format) except Exception as e: - #except (DataTooBigError, HTTPError, ReadTimeout, ConnectionError, InvalidURL, MissingSchema) as e: status = 'request error' - data = {'error':str(e)} + data = {'error': str(e)} headers = {} - - #raise Exception("Request Error: {0}".format(str(e))) finally: if response is not None: response.close() return data, headers, status - def disconnectSocket(self): """Used to hardly disconnect the streaming client""" self.connected = False @@ -1562,12 +1572,87 @@ def disconnectSocket(self): session = self.sessions.pop() session.close() - #self.response.raw._fp.close() - #self.response.close() + # self.response.raw._fp.close() + # self.response.close() + + def postProcessData(self, data, content, response, format): + """ + Post process data, e.g. by extracting links or format conversions + + :param data: + :param content: + :param response: + :return: + """ + # Text + if format == 'text': + data['text'] = content + + # Scrape links + elif format == 'links': + try: + links, base = extractLinks(content, response.url) + data['links'] = links + data['base'] = base + except Exception as e: + data['error'] = 'Could not extract links.' + data['message'] = str(e) + data['response'] = content + + # Get triples + elif format == 'ttl': + try: + data['triples'] = extractTriples(content, 'turtle') + except Exception as e: + data['error'] = 'Could not extract triples.' + data['message'] = str(e) + data['response'] = content + + # Get triples + elif format == 'rdf': + try: + data['triples'] = extractTriples(content, 'xml') + except Exception as e: + data['error'] = 'Could not extract triples.' + data['message'] = str(e) + data['response'] = content + + # Get triples + elif format == 'json-ld': + try: + data['triples'] = extractTriples(content, 'json-ld') + except Exception as e: + data['error'] = 'Could not extract triples.' + data['message'] = str(e) + data['response'] = content + + # JSON + elif format == 'json': + try: + data = json.loads(content) if content != '' else [] + except Exception as e: + data = { + 'error': 'Data could not be converted to JSON', + 'response': content, + 'exception': str(e) + } + + # JSON + elif format == 'xml': + try: + data = xmlToJson(content) + except Exception as e: + data = { + 'error': 'Data could not be converted to JSON', + 'response': content, + 'exception': str(e) + } + + return data @Slot() def captureData(self, nodedata, options=None, logData=None, logMessage=None, logProgress=None): - session_no = options.get('threadnumber',0) + session_no = options.get('threadnumber', 0) self.connected = True # Init pagination @@ -1590,12 +1675,12 @@ def captureData(self, nodedata, options=None, logData=None, logMessage=None, log logMessage("Empty path, node {0} skipped.".format(nodedata['objectid'])) return False - if not urlpath.startswith(('https://','http://','file://')): + if not urlpath.startswith(('https://', 'http://', 'file://')): logMessage("Http or https missing in path, node {0} skipped.".format(nodedata['objectid'])) return False if options['logrequests']: - logpath = self.getLogURL(urlpath,urlparams,options) + logpath = self.getLogURL(urlpath, urlparams, options) logMessage("Capturing data for {0} from {1}".format(nodedata['objectid'], logpath)) # Show browser @@ -1616,16 +1701,16 @@ def captureData(self, nodedata, options=None, logData=None, logMessage=None, log def selectFolder(self): datadir = self.folderEdit.text() - datadir = os.path.dirname(self.mainWindow.settings.value('lastpath', '')) if datadir == '' else datadir - datadir = os.path.expanduser('~') if datadir == '' else datadir - + datadir = os.path.dirname(self.mainWindow.settings.value('lastpath', '')) if datadir == '' else datadir + datadir = os.path.expanduser('~') if datadir == '' else datadir + dlg = SelectFolderDialog(self, 'Select Upload Folder', datadir) if dlg.exec_(): if dlg.optionNodes.isChecked(): - newnodes = [os.path.basename(f) for f in dlg.selectedFiles()] + newnodes = [os.path.basename(f) for f in dlg.selectedFiles()] self.mainWindow.tree.treemodel.addSeedNodes(newnodes) folder = os.path.dirname(dlg.selectedFiles()[0]) - self.folderEdit.setText(folder) + self.folderEdit.setText(folder) else: folder = dlg.selectedFiles()[0] self.folderEdit.setText(folder) @@ -1724,6 +1809,7 @@ def setGlobalOptions(self, settings): if value is not None: self.mainWindow.emptyCheckbox.setChecked(bool(value)) + class AuthTab(ApiTab): """ Module providing authorization @@ -1752,7 +1838,7 @@ def initAuthSetupInputs(self): self.authWidget.setLayout(authlayout) self.authTypeEdit = QComboBox() - self.authTypeEdit.addItems(['Disable','API key','OAuth2', 'Cookie', 'OAuth2 Client Credentials']) + self.authTypeEdit.addItems(['Disable', 'API key', 'OAuth2', 'Cookie', 'OAuth2 Client Credentials']) authlayout.addRow("Authentication type", self.authTypeEdit) self.authURIEdit = QLineEdit() @@ -1776,13 +1862,13 @@ def initAuthSetupInputs(self): authlayout.addRow("Scopes", self.scopeEdit) self.proxyEdit = QLineEdit() - self.proxyEdit .setToolTip(wraptip("The proxy will be used for fetching data only, not for the login procedure.")) + self.proxyEdit.setToolTip( + wraptip("The proxy will be used for fetching data only, not for the login procedure.")) authlayout.addRow("Proxy", self.proxyEdit) - @Slot() def editAuthSettings(self): - dialog = QDialog(self,Qt.WindowSystemMenuHint | Qt.WindowTitleHint) + dialog = QDialog(self, Qt.WindowSystemMenuHint | Qt.WindowTitleHint) dialog.setWindowTitle("Authentication settings") dialog.setMinimumWidth(400) @@ -1819,7 +1905,7 @@ def apply(): dialog.close() - #connect the nested functions above to the dialog-buttons + # connect the nested functions above to the dialog-buttons buttons.accepted.connect(apply) buttons.rejected.connect(close) dialog.exec_() @@ -1837,9 +1923,10 @@ def initLoginInputs(self, toggle=True): loginlayout.addWidget(QLabel("Name")) self.tokenNameEdit = QLineEdit() - self.tokenNameEdit.setToolTip(wraptip("The name of the access token parameter or the authorization header. If you select an authentication method different from API key (e.g. OAuth2 or Cookie), name the is overriden by the selected method.")) + self.tokenNameEdit.setToolTip(wraptip( + "The name of the access token parameter or the authorization header. If you select an authentication method different from API key (e.g. OAuth2 or Cookie), name the is overriden by the selected method.")) # If you leave this empty, the default value is 'access_token' for param-method and 'Authorization' for header-method. - loginlayout.addWidget(self.tokenNameEdit,1) + loginlayout.addWidget(self.tokenNameEdit, 1) rowcaption = "Authorization" loginlayout.addWidget(QLabel("Access token")) @@ -1848,7 +1935,7 @@ def initLoginInputs(self, toggle=True): self.tokenEdit = QLineEdit() self.tokenEdit.setEchoMode(QLineEdit.Password) - loginlayout.addWidget(self.tokenEdit,2) + loginlayout.addWidget(self.tokenEdit, 2) self.authButton = QPushButton('Settings', self) self.authButton.clicked.connect(self.editAuthSettings) @@ -1860,7 +1947,7 @@ def initLoginInputs(self, toggle=True): self.loginButton.clicked.connect(self.doLogin) loginlayout.addWidget(self.loginButton) - #self.mainLayout.addRow(rowcaption, loginwidget) + # self.mainLayout.addRow(rowcaption, loginwidget) self.extraLayout.addRow(rowcaption, loginlayout) def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' @@ -1869,7 +1956,9 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' # Auth type try: - options['auth_type'] = self.authTypeEdit.currentText().strip() if self.authTypeEdit.currentText() != "" else defaults.get('auth_type', '') + options[ + 'auth_type'] = self.authTypeEdit.currentText().strip() if self.authTypeEdit.currentText() != "" else defaults.get( + 'auth_type', '') except AttributeError: options['auth_type'] = defaults.get('auth_type', '') @@ -1897,28 +1986,28 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' options['auth_tokenname'] = self.tokenNameEdit.text() except AttributeError: - options.pop('auth_tokenname',None) + options.pop('auth_tokenname', None) options['auth'] = defaults.get('auth', 'disable') # Override authorization settings (token handling) # based on authentication settings if options.get('auth_type') == 'OAuth2': - #options['auth'] = 'header' + # options['auth'] = 'header' options['auth_prefix'] = "Bearer " - #options['auth_tokenname'] = "Authorization" + # options['auth_tokenname'] = "Authorization" elif options.get('auth_type') == 'OAuth2 Client Credentials': - #options['auth'] = 'header' + # options['auth'] = 'header' options['auth_prefix'] = "Bearer " - #options['auth_tokenname'] = "Authorization" + # options['auth_tokenname'] = "Authorization" elif options.get('auth_type') == 'OAuth1': - #options['auth'] = 'disable' # managed by Twitter module + # options['auth'] = 'disable' # managed by Twitter module options['auth_prefix'] = '' - #options['auth_tokenname'] = '' + # options['auth_tokenname'] = '' elif options.get('auth_type') == 'Cookie': - #options['auth'] = 'header' + # options['auth'] = 'header' options['auth_prefix'] = '' - #options['auth_tokenname'] = 'Cookie' + # options['auth_tokenname'] = 'Cookie' if options['auth'] == 'disable': options['auth_prefix'] = '' @@ -1926,9 +2015,8 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' return options - # Transfer options to GUI - def setSettings(self, settings = {}): + def setSettings(self, settings={}): settings = super(AuthTab, self).setSettings(settings) # Merge options @@ -1980,7 +2068,7 @@ def fetchData(self, nodedata, options=None, logData=None, logMessage=None, logPr if not self.auth_userauthorized and self.auth_preregistered: raise Exception('You are not authorized, login please!') - session_no = options.get('threadnumber',0) + session_no = options.get('threadnumber', 0) self.closeSession(session_no) self.connected = True self.speed = options.get('speed', None) @@ -2005,7 +2093,6 @@ def fetchData(self, nodedata, options=None, logData=None, logMessage=None, logPr # Save page options['currentpage'] = page - # build url method, urlpath, urlparams, payload, requestheaders = self.buildUrl(nodedata, options, logProgress) @@ -2013,17 +2100,17 @@ def fetchData(self, nodedata, options=None, logData=None, logMessage=None, logPr logMessage("Empty path, node {0} skipped.".format(nodedata['objectid'])) return False - if not urlpath.startswith(('https://','http://','file://')): + if not urlpath.startswith(('https://', 'http://', 'file://')): logMessage("Http or https missing in path, node {0} skipped.".format(nodedata['objectid'])) return False if options['logrequests']: - logpath = self.getLogURL(urlpath,urlparams,options) + logpath = self.getLogURL(urlpath, urlparams, options) logMessage("Fetching data for {0} from {1}".format(nodedata['objectid'], logpath)) # data options['querytime'] = str(datetime.now()) - data, headers, status = self.request(session_no,urlpath, urlparams, requestheaders, method, payload, + data, headers, status = self.request(session_no, urlpath, urlparams, requestheaders, method, payload, foldername, filename, fileext, format=format) # status handling @@ -2051,10 +2138,8 @@ def fetchData(self, nodedata, options=None, logData=None, logMessage=None, logPr return True - - @Slot() - def doLogin(self, session_no = 0): + def doLogin(self, session_no=0): """ Show login window :param session_no: the number of the session used for login @@ -2070,16 +2155,19 @@ def doLogin(self, session_no = 0): elif options['auth_type'] == 'Cookie': self.doCookieLogin(session_no) elif options['auth_type'] == 'API key': - QMessageBox.information(self, "Facepager", "Manually enter your API key into the access token field or change the authentication method in the settings.") + QMessageBox.information(self, "Facepager", + "Manually enter your API key into the access token field or change the authentication method in the settings.") elif options['auth_type'] == 'Disable': - QMessageBox.information(self, "Login disabled","No authentication method selected. Please choose a method in the settings.", QMessageBox.StandardButton.Ok) + QMessageBox.information(self, "Login disabled", + "No authentication method selected. Please choose a method in the settings.", + QMessageBox.StandardButton.Ok) elif options['auth_type'] == 'OAuth2 External': self.doOAuth2ExternalLogin(session_no) else: self.doOAuth2Login(session_no) @Slot() - def doOAuth1Login(self, session_no = 0): + def doOAuth1Login(self, session_no=0): try: # use credentials from input if provided clientid = self.getClientId() @@ -2130,7 +2218,7 @@ def doOAuth2Login(self, session_no=0): loginurl, loginparams = self.parseURL(loginurl) params.update(loginparams) - urlpath, urlparams, templateparams = self.getURL(loginurl,params,{},{}) + urlpath, urlparams, templateparams = self.getURL(loginurl, params, {}, {}) url = urlpath + '?' + urllib.parse.urlencode(urlparams) self.showLoginWindow(url) @@ -2159,7 +2247,7 @@ def doOAuth2ExternalLogin(self, session_no=0): raise Exception('Client Id is missing, please adjust settings!') self.startLoginServer(0) - redirect_uri = "http://localhost:"+str(self.loginServerInstance.server_port) + redirect_uri = "http://localhost:" + str(self.loginServerInstance.server_port) params = {'client_id': clientid, 'redirect_uri': redirect_uri, @@ -2178,7 +2266,6 @@ def doOAuth2ExternalLogin(self, session_no=0): str(e), QMessageBox.StandardButton.Ok) - @Slot() def doCookieLogin(self, session_no=0): def newCookie(domain, cookie): @@ -2187,7 +2274,7 @@ def newCookie(domain, cookie): try: options = self.getSettings() - url= options.get('auth_uri', '') + url = options.get('auth_uri', '') if url == '': raise Exception('Login URL is missing, please adjust settings!') @@ -2213,20 +2300,19 @@ def doTwitterAppLogin(self, session_no=0): try: # See https://developer.twitter.com/en/docs/basics/authentication/overview/application-only self.auth_preregistered = False - clientid = self.clientIdEdit.text() # no defaults + clientid = self.clientIdEdit.text() # no defaults if clientid == '': raise Exception('Client Id is missing, please adjust settings!') - clientsecret = self.clientSecretEdit.text() # no defaults + clientsecret = self.clientSecretEdit.text() # no defaults if clientsecret == '': raise Exception('Client Secret is missing, please adjust settings!') options = self.getSettings() - path= options.get('auth_uri', '') + path = options.get('auth_uri', '') if path == '': raise Exception('Login URL is missing, please adjust settings!') - basicauth = urllib.parse.quote_plus(clientid) + ':' + urllib.parse.quote_plus(clientsecret) basicauth = base64.b64encode(basicauth.encode('utf-8')).decode('utf-8') @@ -2254,7 +2340,6 @@ def doTwitterAppLogin(self, session_no=0): str(e), QMessageBox.StandardButton.Ok) - @Slot() def showLoginWindow(self, url=''): self.loginWindow = LoginWebDialog( @@ -2337,7 +2422,7 @@ def initSession(self, no=0, renew=False): else: return self.initOAuth2Session(no, renew) - def initOAuth1Session(self,no=0, renew=False): + def initOAuth1Session(self, no=0, renew=False): """ Return session or create if necessary :param no: session number @@ -2362,7 +2447,7 @@ def initOAuth2Session(self, no=0, renew=False): return super(AuthTab, self).initSession(no, renew) def getOAuth1Service(self): - if not hasattr(self,'oauthdata'): + if not hasattr(self, 'oauthdata'): self.oauthdata = {} service = OAuth1Service( @@ -2392,7 +2477,6 @@ def getOAuth1Service(self): return service - def getOAuth1Token(self, url): success = False url = urllib.parse.parse_qs(url) @@ -2414,7 +2498,6 @@ def getOAuth1Token(self, url): success = True return success - def getOAuth2Token(self, url): success = False try: @@ -2422,7 +2505,7 @@ def getOAuth2Token(self, url): # Parse URL urlparsed = urlparse(url) - query = dict(parse_qsl(urlparsed.query)) # dict & parse_qsl to get single values instead of lists + query = dict(parse_qsl(urlparsed.query)) # dict & parse_qsl to get single values instead of lists fragment = dict(parse_qsl(urlparsed.fragment)) # dict & parse_qsl to get single values instead of lists # Get code or token @@ -2441,7 +2524,6 @@ def getOAuth2Token(self, url): pass success = True - # Flow: response_type=code if url.startswith(options['redirect_uri']) and code is not None: try: @@ -2452,7 +2534,7 @@ def getOAuth2Token(self, url): scope = self.scopeEdit.text() if self.scopeEdit.text() != "" else \ self.defaults.get('scope', None) - headers = options.get("headers",{}) + headers = options.get("headers", {}) headers = {key.lower(): value for (key, value) in headers.items()} session = OAuth2Session(clientid, redirect_uri=options['redirect_uri'], scope=scope) @@ -2466,7 +2548,7 @@ def getOAuth2Token(self, url): # Check access and set token self.checkPreregisteredAccess(token) - self.tokenEdit.setText(token.get('access_token','')) + self.tokenEdit.setText(token.get('access_token', '')) try: self.authEdit.setCurrentIndex(self.authEdit.findText('header')) except AttributeError: @@ -2519,7 +2601,7 @@ def authorizeUser(self, userid): return False # App salt - salt = getDictValueOrNone(credentials,'facepager.salt') + salt = getDictValueOrNone(credentials, 'facepager.salt') if salt is None: self.auth_userauthorized = False return False @@ -2538,7 +2620,7 @@ def authorizeUser(self, userid): self.auth_userauthorized = False return False - authurl += '?module='+self.name.lower()+'&usertoken='+usertoken.hex() + authurl += '?module=' + self.name.lower() + '&usertoken=' + usertoken.hex() session = self.initOAuth2Session(0, True) data, headers, status = self.request(0, authurl) self.closeSession(0) @@ -2566,12 +2648,12 @@ class GenericTab(AuthTab): def __init__(self, mainWindow=None): super(GenericTab, self).__init__(mainWindow, "Generic") - #Defaults + # Defaults self.timeout = 60 self.defaults['basepath'] = '' # Standard inputs - self.initInputs() + self.initBasicInputs() # Header, Verbs self.initHeaderInputs() @@ -2580,7 +2662,7 @@ def __init__(self, mainWindow=None): # Extract input self.initPagingInputs(True) - self.initResponseInputs(True) + self.initResponseInputs() self.initFileInputs() @@ -2591,12 +2673,11 @@ def __init__(self, mainWindow=None): self.loadDoc() self.loadSettings() - def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' options = super(GenericTab, self).getSettings(purpose) if purpose != 'preset': - options['querytype'] = self.name + ':'+options['basepath']+options['resource'] + options['querytype'] = self.name + ':' + options['basepath'] + options['resource'] return options @@ -2606,6 +2687,103 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' # self.logmessage.emit("SSL certificate error ignored: %s (Warning: Your connection might be insecure!)" % url) +class SparqlTab(AuthTab): + def __init__(self, mainWindow=None): + super(SparqlTab, self).__init__(mainWindow, "SPARQL") + + # Defaults + self.timeout = 60 + self.defaults['basepath'] = 'https://query.wikidata.org/sparql' + self.defaults['params'] = {'query': 'SELECT * WHERE {?s ?p ?o} LIMIT 100'} + + # Inputs + self.initBasepathInput("Endpoint") + self.initQueryInput("Query") + self.initPagingInputs(True) + self.initResponseInputs(True) + + # Load options and settings + self.loadDoc() + self.loadSettings() + + def initQueryInput(self, rowLabel="Query"): + # Payload + self.queryWidget = QWidget() + self.queryLayout = QHBoxLayout() + self.queryLayout.setContentsMargins(0, 0, 0, 0) + self.queryWidget.setLayout(self.queryLayout) + + self.queryEdit = QPlainTextEdit() + self.queryEdit.setLineWrapMode(QPlainTextEdit.NoWrap) + self.queryLayout.addWidget(self.queryEdit) + self.queryLayout.setStretch(0, 1); + + self.queryEditButton = QToolButton(self) + self.queryEditButton.setText("..") + self.queryEditButton.clicked.connect(self.openQueryEditDialog) + self.queryLayout.addWidget(self.queryEditButton) + self.queryLayout.setStretch(2, 1); + + self.mainLayout.addRow(rowLabel, self.queryWidget) + + queryValue = getDictValue(self.defaults, 'params.query') + self.queryEdit.setPlainText(queryValue) + + def openQueryEditDialog(self): + dialog = EditValueDialog(self) + dialog.show(self.queryEdit) + + def getSettings(self, purpose='fetch'): + """ + Get the settings + + :param purpose: 'fetch' for fetching data, + 'settings' for saving settings when closing the app, + 'preset' for saving settings to a preset + :return: + """ + options = super(SparqlTab, self).getSettings(purpose) + + # We don't have a resource input in the SPARQL tab. + # Thus, provide hardwired values. And set the user-agent + options['resource'] = '' + options['headers'] = {'User-Agent': settings.get('userAgent')} + + # Get the query value + queryValue = self.queryEdit.toPlainText() + + if purpose != 'preset': + # Save the endpoint so we see it later in the exports + options['querytype'] = self.name + ':' + options['basepath'] + + # Add a LIMIT if not set + if not re.search(r'\bLIMIT\b\s+\d+\s*$', queryValue, re.IGNORECASE): + queryValue += " LIMIT 100" + + # Set query parameters + options['params'] = { + 'query': queryValue, + 'format': self.formatEdit.currentText().strip() + } + + return options + + def setSettings(self, settings={}): + """ + Populates input fields from loaded options and presets + :param settings: Dict with options + :return: + """ + + settings = super(SparqlTab, self).setSettings(settings) + + # Insert the query parameter value into the query box + queryValue = getDictValue(settings, 'params.query') + self.queryEdit.setPlainText(queryValue) + + return settings + + class FacebookTab(AuthTab): def __init__(self, mainWindow=None): super(FacebookTab, self).__init__(mainWindow, "Facebook") @@ -2613,9 +2791,9 @@ def __init__(self, mainWindow=None): # Authorization self.auth_userauthorized = False - #Defaults + # Defaults self.defaults['auth_type'] = "OAuth2" - self.defaults['scope'] = '' #user_groups + self.defaults['scope'] = '' # user_groups self.defaults['basepath'] = 'https://graph.facebook.com/v3.4' self.defaults['resource'] = '/' self.defaults['auth_uri'] = 'https://www.facebook.com/dialog/oauth' @@ -2624,7 +2802,7 @@ def __init__(self, mainWindow=None): self.defaults['login_buttoncaption'] = " Login to Facebook " # Query Box - self.initInputs() + self.initBasicInputs() # Pages Box self.initPagingInputs() @@ -2637,7 +2815,7 @@ def __init__(self, mainWindow=None): def initAuthSettingsInputs(self): authlayout = QFormLayout() - authlayout.setContentsMargins(0,0,0,0) + authlayout.setContentsMargins(0, 0, 0, 0) self.authWidget.setLayout(authlayout) self.pageIdEdit = QLineEdit() @@ -2648,7 +2826,7 @@ def initAuthSettingsInputs(self): authlayout.addRow("Client Id", self.clientIdEdit) self.scopeEdit = QLineEdit() - authlayout.addRow("Scopes",self.scopeEdit) + authlayout.addRow("Scopes", self.scopeEdit) def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' options = super(FacebookTab, self).getSettings(purpose) @@ -2662,7 +2840,7 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' return options - def setSettings(self, settings ={}): + def setSettings(self, settings={}): settings = super(FacebookTab, self).setSettings(settings) if 'pageid' in settings: @@ -2675,7 +2853,7 @@ def fetchData(self, nodedata, options=None, logData=None, logMessage=None, logPr if not self.auth_userauthorized and self.auth_preregistered: raise Exception('You are not authorized, login please!') - if options.get('access_token','') == '': + if options.get('access_token', '') == '': raise Exception('Access token is missing, login please!') self.connected = True @@ -2708,7 +2886,7 @@ def fetchData(self, nodedata, options=None, logData=None, logMessage=None, logPr # data options['querytime'] = str(datetime.now()) - data, headers, status = self.request(session_no,urlpath, urlparams) + data, headers, status = self.request(session_no, urlpath, urlparams) options['ratelimit'] = False options['querystatus'] = status @@ -2730,20 +2908,20 @@ def fetchData(self, nodedata, options=None, logData=None, logMessage=None, logPr options['info'] = {'x-app-usage': msg.format(self.appusage)} if (status != "fetched (200)"): - msg = getDictValue(data,"error.message") - code = getDictValue(data,"error.code") + msg = getDictValue(data, "error.message") + code = getDictValue(data, "error.code") logMessage("Error '{0}' for {1} with message {2}.".format(status, nodedata['objectid'], msg)) # see https://developers.facebook.com/docs/graph-api/using-graph-api # see https://developers.facebook.com/docs/graph-api/advanced/rate-limiting/ - if (code in ['4','17','32','613']) and (status in ['error (400)', 'error (403)']): + if (code in ['4', '17', '32', '613']) and (status in ['error (400)', 'error (403)']): options['ratelimit'] = True else: options['ratelimit'] = False logData(data, options, headers) if logProgress is not None: - logProgress({'page':page+1}) + logProgress({'page': page + 1}) # paging options = self.updatePagingOptions(data, options) @@ -2759,19 +2937,19 @@ def fetchData(self, nodedata, options=None, logData=None, logMessage=None, logPr break @Slot() - def doLogin(self, session_no = 0): + def doLogin(self, session_no=0): try: - #use credentials from input if provided + # use credentials from input if provided clientid = self.getClientId() if clientid is None: return False - scope= self.scopeEdit.text() if self.scopeEdit.text() != "" else self.defaults.get('scope','') - + scope = self.scopeEdit.text() if self.scopeEdit.text() != "" else self.defaults.get('scope', '') - url = self.defaults['auth_uri'] +"?client_id=" + clientid + "&redirect_uri="+self.defaults['redirect_uri']+"&response_type=token&scope="+scope+"&display=popup" + url = self.defaults['auth_uri'] + "?client_id=" + clientid + "&redirect_uri=" + self.defaults[ + 'redirect_uri'] + "&response_type=token&scope=" + scope + "&display=popup" self.showLoginWindow(url) except Exception as e: - QMessageBox.critical(self, "Login canceled",str(e),QMessageBox.StandardButton.Ok) + QMessageBox.critical(self, "Login canceled", str(e), QMessageBox.StandardButton.Ok) def getUserId(self, token): data, headers, status = self.request( @@ -2786,7 +2964,7 @@ def getUserId(self, token): def onLoginWindowChanged(self, url): if "#access_token" in url.toString(): try: - url = urllib.parse.urlparse(url.toString(),allow_fragments=True) + url = urllib.parse.urlparse(url.toString(), allow_fragments=True) fragment = urllib.parse.parse_qs(url.fragment) token = fragment.get('access_token').pop() @@ -2803,17 +2981,18 @@ def onLoginWindowChanged(self, url): # Get page access token pageid = self.pageIdEdit.text().strip() if pageid != '': - data, headers, status = self.request(None, self.basepathEdit.currentText().strip()+'/'+pageid+'?fields=access_token&scope=pages_show_list&access_token='+token) + data, headers, status = self.request(None, + self.basepathEdit.currentText().strip() + '/' + pageid + '?fields=access_token&scope=pages_show_list&access_token=' + token) if status != 'fetched (200)': raise Exception("Could not authorize for page. Check page ID in the settings.") - token = data.get('access_token','') + token = data.get('access_token', '') # Set token self.tokenEdit.setText(token) except Exception as e: - QMessageBox.critical(self,"Login error", - str(e),QMessageBox.StandardButton.Ok) + QMessageBox.critical(self, "Login error", + str(e), QMessageBox.StandardButton.Ok) self.closeLoginWindow() @@ -2828,7 +3007,7 @@ def __init__(self, mainWindow=None, name='Amazon'): self.defaults['format'] = 'xml' # Standard inputs - self.initInputs() + self.initBasicInputs() # Header, Verbs self.initHeaderInputs() @@ -2836,7 +3015,7 @@ def __init__(self, mainWindow=None, name='Amazon'): self.initUploadFolderInput() # Extract input - self.initResponseInputs(True) + self.initResponseInputs() # Pages Box self.initPagingInputs(True) @@ -2847,7 +3026,6 @@ def __init__(self, mainWindow=None, name='Amazon'): self.loadDoc() self.loadSettings() - def initLoginInputs(self): # token and login button loginwidget = QWidget() @@ -2878,18 +3056,21 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' options = super(AmazonTab, self).getSettings(purpose) options['auth'] = 'disable' - #options['format'] = self.defaults.get('format', '') - options['service'] = self.serviceEdit.text().strip() if self.serviceEdit.text() != "" else self.defaults.get('service', '') + # options['format'] = self.defaults.get('format', '') + options['service'] = self.serviceEdit.text().strip() if self.serviceEdit.text() != "" else self.defaults.get( + 'service', '') options['region'] = self.regionEdit.text().strip() if self.regionEdit.text() != "" else self.defaults.get( 'region', '') if purpose != 'preset': - options['secretkey'] = self.secretkeyEdit.text().strip() #if self.secretkeyEdit.text() != "" else self.defaults.get('auth_uri', '') - options['accesskey'] = self.accesskeyEdit.text().strip() #if self.accesskeyEdit.text() != "" else self.defaults.get('redirect_uri', '') + options[ + 'secretkey'] = self.secretkeyEdit.text().strip() # if self.secretkeyEdit.text() != "" else self.defaults.get('auth_uri', '') + options[ + 'accesskey'] = self.accesskeyEdit.text().strip() # if self.accesskeyEdit.text() != "" else self.defaults.get('redirect_uri', '') return options - def setSettings(self, settings = {}): + def setSettings(self, settings={}): settings = super(AmazonTab, self).setSettings(settings) if 'secretkey' in settings: @@ -2903,7 +3084,6 @@ def setSettings(self, settings = {}): return settings - # Get authorization header # See https://docs.aws.amazon.com/de_de/general/latest/gr/sigv4-signed-request-examples.html def signRequest(self, urlpath, urlparams, headers, method, payload, options): @@ -2918,7 +3098,6 @@ def signRequest(self, urlpath, urlparams, headers, method, payload, options): if access_key == '' or secret_key == '': raise Exception('Access key or secret key is missing, please fill the input fields!') - # Key derivation functions. See: # http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python def sign(key, msg): @@ -2983,7 +3162,7 @@ def getSignatureKey(key, dateStamp, regionName, serviceName): payload_buffer = payload payload = payload_buffer.read() payload_buffer.rewind() - #payload = payload.decode('utf-8') + # payload = payload.decode('utf-8') payload_hash = hashlib.sha256(payload).hexdigest() @@ -3022,6 +3201,7 @@ def getSignatureKey(key, dateStamp, regionName, serviceName): return (headers) + class TwitterTab(AuthTab): def __init__(self, mainWindow=None): super(TwitterTab, self).__init__(mainWindow, "Twitter") @@ -3033,7 +3213,7 @@ def __init__(self, mainWindow=None): self.defaults['basepath'] = 'https://api.twitter.com/1.1' self.defaults['resource'] = '/search/tweets' self.defaults['params'] = {'q': ''} - #self.defaults['extension'] = ".json" + # self.defaults['extension'] = ".json" self.defaults['auth_type'] = 'OAuth1' self.defaults['access_token_url'] = 'https://api.twitter.com/oauth/access_token' @@ -3042,7 +3222,7 @@ def __init__(self, mainWindow=None): self.defaults['login_window_caption'] = 'Twitter Login Page' # Query and Parameter Box - self.initInputs() + self.initBasicInputs() self.initPagingInputs() self.initAuthSetupInputs() @@ -3072,7 +3252,6 @@ def initLoginInputs(self): self.loginButton.clicked.connect(self.doLogin) loginlayout.addWidget(self.loginButton) - # Add to main-Layout self.extraLayout.addRow("Access Token", loginlayout) @@ -3081,7 +3260,7 @@ def initAuthSetupInputs(self): authlayout.setContentsMargins(0, 0, 0, 0) self.authWidget.setLayout(authlayout) - self.authTypeEdit= QComboBox() + self.authTypeEdit = QComboBox() self.authTypeEdit.addItems(['OAuth1', 'OAuth2 Client Credentials']) authlayout.addRow("Authentication type", self.authTypeEdit) @@ -3107,8 +3286,8 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' def getUserId(self, session): # Send request response = session.request('GET', 'https://api.twitter.com/1.1/account/settings.json', timeout=self.timeout) - if not response.ok : - return None + if not response.ok: + return None data = response.json() if response.text != '' else [] return getDictValueOrNone(data, 'screen_name') @@ -3122,7 +3301,7 @@ def fetchData(self, nodedata, options=None, logData=None, logMessage=None, logPr self.speed = options.get('speed', None) self.timeout = options.get('timeout', 15) self.maxsize = options.get('maxsize', 5) - session_no = options.get('threadnumber',0) + session_no = options.get('threadnumber', 0) # Init pagination options = self.initPagingOptions(nodedata, options) @@ -3180,7 +3359,6 @@ def fetchData(self, nodedata, options=None, logData=None, logMessage=None, logPr break - class TwitterStreamingTab(TwitterTab): def __init__(self, mainWindow=None): super(TwitterTab, self).__init__(mainWindow, "Twitter Streaming") @@ -3197,13 +3375,13 @@ def __init__(self, mainWindow=None): self.defaults['basepath'] = 'https://stream.twitter.com/1.1' self.defaults['resource'] = '/statuses/filter' self.defaults['params'] = {'track': ''} - #self.defaults['extension'] = ".json" + # self.defaults['extension'] = ".json" self.defaults['key_objectid'] = 'id' self.defaults['key_nodedata'] = None # Query Box - self.initInputs() + self.initBasicInputs() self.initAuthSetupInputs() self.initLoginInputs() @@ -3215,8 +3393,8 @@ def __init__(self, mainWindow=None): def stream(self, session_no=0, path='', args=None, headers=None): self.connected = True - self.retry_counter=0 - self.last_reconnect=QDateTime.currentDateTime() + self.retry_counter = 0 + self.last_reconnect = QDateTime.currentDateTime() try: session = self.initSession(session_no) @@ -3226,34 +3404,36 @@ def _send(): try: if headers is not None: response = session.post(path, params=args, - headers=headers, - timeout=self.timeout, - stream=True) + headers=headers, + timeout=self.timeout, + stream=True) else: response = session.get(path, params=args, timeout=self.timeout, - stream=True) + stream=True) except requests.exceptions.Timeout: raise Exception('Request timed out.') else: if response.status_code != 200: - if self.retry_counter<=5: - self.logMessage("Reconnecting in 3 Seconds: " + str(response.status_code) + ". Message: "+ str(response.content)) + if self.retry_counter <= 5: + self.logMessage( + "Reconnecting in 3 Seconds: " + str(response.status_code) + ". Message: " + str( + response.content)) time.sleep(3) - if self.last_reconnect.secsTo(QDateTime.currentDateTime())>120: + if self.last_reconnect.secsTo(QDateTime.currentDateTime()) > 120: self.retry_counter = 0 _send() else: - self.retry_counter+=1 + self.retry_counter += 1 _send() else: - #self.connected = False + # self.connected = False self.disconnectSocket() - raise Exception("Request Error: " + str(response.status_code) + ". Message: "+str(response.content)) + raise Exception("Request Error: " + str(response.status_code) + ". Message: " + str( + response.content)) return response - while self.connected: response = _send() if response: @@ -3276,8 +3456,8 @@ def _send(): response.close() except AttributeError: - #This exception is thrown when canceling the connection - #Only re-raise if not manually canceled + # This exception is thrown when canceling the connection + # Only re-raise if not manually canceled if self.connected: raise finally: @@ -3299,11 +3479,11 @@ def fetchData(self, nodedata, options=None, logData=None, logMessage=None, logPr logpath = self.getLogURL(urlpath, urlparams, options) logMessage("Fetching data for {0} from {1}".format(nodedata['objectid'], logpath)) - self.timeout = options.get('timeout',30) + self.timeout = options.get('timeout', 30) self.maxsize = options.get('maxsize', 5) # data - session_no = options.get('threadnumber',0) + session_no = options.get('threadnumber', 0) for data, headers, status in self.stream(session_no, path=urlpath, args=urlparams): # data options['querytime'] = str(datetime.now()) @@ -3311,6 +3491,7 @@ def fetchData(self, nodedata, options=None, logData=None, logMessage=None, logPr logData(data, options, headers) + class YoutubeTab(AuthTab): def __init__(self, mainWindow=None): @@ -3324,7 +3505,8 @@ def __init__(self, mainWindow=None): self.defaults['auth_uri'] = 'https://accounts.google.com/o/oauth2/auth' self.defaults['token_uri'] = "https://accounts.google.com/o/oauth2/token" self.defaults['redirect_uri'] = 'https://localhost' - self.defaults['scope'] = "https://www.googleapis.com/auth/youtube https://www.googleapis.com/auth/youtube.readonly https://www.googleapis.com/auth/youtube.force-ssl" + self.defaults[ + 'scope'] = "https://www.googleapis.com/auth/youtube https://www.googleapis.com/auth/youtube.readonly https://www.googleapis.com/auth/youtube.force-ssl" self.defaults['response_type'] = "code" self.defaults['login_buttoncaption'] = " Login to Google " @@ -3333,10 +3515,10 @@ def __init__(self, mainWindow=None): self.defaults['auth'] = 'param' self.defaults['basepath'] = "https://www.googleapis.com/youtube/v3" self.defaults['resource'] = '/search' - self.defaults['params'] = {'q':'','part':'snippet','maxResults':'50'} + self.defaults['params'] = {'q': '', 'part': 'snippet', 'maxResults': '50'} # Standard inputs - self.initInputs() + self.initBasicInputs() # Pages Box self.initPagingInputs() @@ -3350,11 +3532,11 @@ def __init__(self, mainWindow=None): def initAuthSetupInputs(self): authlayout = QFormLayout() - authlayout.setContentsMargins(0,0,0,0) + authlayout.setContentsMargins(0, 0, 0, 0) self.authWidget.setLayout(authlayout) - self.authTypeEdit= QComboBox() - self.authTypeEdit.addItems(['OAuth2', 'OAuth2 External','API key']) + self.authTypeEdit = QComboBox() + self.authTypeEdit.addItems(['OAuth2', 'OAuth2 External', 'API key']) authlayout.addRow("Authentication type", self.authTypeEdit) self.clientIdEdit = QLineEdit() @@ -3366,16 +3548,16 @@ def initAuthSetupInputs(self): authlayout.addRow("Client Secret", self.clientSecretEdit) self.scopeEdit = QLineEdit() - authlayout.addRow("Scopes",self.scopeEdit) + authlayout.addRow("Scopes", self.scopeEdit) def getUserId(self, token): data, headers, status = self.request( - None, 'https://www.googleapis.com/youtube/v3/channels?mine=true&access_token='+token) + None, 'https://www.googleapis.com/youtube/v3/channels?mine=true&access_token=' + token) if status != 'fetched (200)': return None - return getDictValueOrNone(data,'items.0.id') + return getDictValueOrNone(data, 'items.0.id') def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' options = super(YoutubeTab, self).getSettings(purpose) @@ -3384,7 +3566,7 @@ def getSettings(self, purpose='fetch'): # purpose = 'fetch'|'settings'|'preset' options['auth'] = 'param' options['auth_prefix'] = '' options['auth_tokenname'] = 'key' - else: # OAuth2 + else: # OAuth2 options['auth'] = 'header' options['auth_prefix'] = 'Bearer ' options['auth_tokenname'] = 'Authorization' @@ -3447,5 +3629,6 @@ def send(self, req, **kwargs): # pylint: disable=unused-argument def close(self): pass + class DataTooBigError(Exception): - pass \ No newline at end of file + pass diff --git a/src/dialogs/presets.py b/src/dialogs/presets.py index 886d06e5..2792b640 100644 --- a/src/dialogs/presets.py +++ b/src/dialogs/presets.py @@ -14,7 +14,9 @@ import platform from widgets.dictionarytree import DictionaryTree from widgets.progressbar import ProgressBar -from utilities import wraptip, formatdict +from utilities import * +from settings import * + class PresetWindow(QDialog): logmessage = Signal(str) @@ -26,28 +28,28 @@ class PresetWindow(QDialog): progressStop = Signal() def __init__(self, parent=None): - super(PresetWindow,self).__init__(parent) + super(PresetWindow, self).__init__(parent) self.mainWindow = parent self.setWindowTitle("Presets") self.setMinimumWidth(800); self.setMinimumHeight(600); - #layout + # layout layout = QVBoxLayout(self) self.setLayout(layout) - #loading indicator + # loading indicator self.loadingLock = threading.Lock() self.loadingIndicator = QLabel('Loading...please wait a second.') self.loadingIndicator.hide() layout.addWidget(self.loadingIndicator) - #Middle + # Middle central = QSplitter(self) - layout.addWidget(central,1) + layout.addWidget(central, 1) - #list view + # list view self.presetList = QTreeWidget(self) self.presetList.setHeaderHidden(True) self.presetList.setColumnCount(1) @@ -63,30 +65,33 @@ def __init__(self, parent=None): self.categoryWidget.setPalette(p) self.categoryWidget.setAutoFillBackground(True) + self.pipelineLayout = QVBoxLayout() + # self.pipelineLayout.setContentsMargins(0, 0, 0, 0) + self.categoryWidget.setLayout(self.pipelineLayout) - self.categoryLayout=QVBoxLayout() - #self.categoryLayout.setContentsMargins(0, 0, 0, 0) - self.categoryWidget.setLayout(self.categoryLayout) - - self.categoryView=QScrollArea() - self.categoryView.setWidgetResizable(True) - self.categoryView.setWidget(self.categoryWidget) - central.addWidget(self.categoryView) + self.pipelineView = QScrollArea() + self.pipelineView.setWidgetResizable(True) + self.pipelineView.setWidget(self.categoryWidget) + central.addWidget(self.pipelineView) central.setStretchFactor(1, 2) # Pipeline header self.pipelineName = QLabel('') self.pipelineName.setWordWrap(True) self.pipelineName.setStyleSheet("QLabel {font-size:15pt;}") - self.categoryLayout.addWidget(self.pipelineName) + self.pipelineLayout.addWidget(self.pipelineName) + + # Pipeline description + self.pipelineDescription = TextViewer() + self.pipelineLayout.addWidget(self.pipelineDescription) # Pipeline items - # self.pipelineWidget = QTreeWidget() - # self.pipelineWidget.setIndentation(0) - # self.pipelineWidget.setUniformRowHeights(True) - # self.pipelineWidget.setColumnCount(4) - # self.pipelineWidget.setHeaderLabels(['Name','Module','Basepath','Resource']) - # self.categoryLayout.addWidget(self.pipelineWidget) + self.pipelineWidget = QTreeWidget() + self.pipelineWidget.setIndentation(0) + self.pipelineWidget.setUniformRowHeights(True) + self.pipelineWidget.setColumnCount(4) + self.pipelineWidget.setHeaderLabels(['Name', 'Module', 'Basepath', 'Resource']) + self.pipelineLayout.addWidget(self.pipelineWidget) # preset widget self.presetWidget = QWidget() @@ -96,16 +101,16 @@ def __init__(self, parent=None): self.presetWidget.setPalette(p) self.presetWidget.setAutoFillBackground(True) - #self.presetWidget.setStyleSheet("background-color: rgb(255,255,255);") - self.presetView=QScrollArea() + # self.presetWidget.setStyleSheet("background-color: rgb(255,255,255);") + self.presetView = QScrollArea() self.presetView.setWidgetResizable(True) self.presetView.setWidget(self.presetWidget) central.addWidget(self.presetView) central.setStretchFactor(2, 2) - #self.detailView.setFrameStyle(QFrame.Box) - self.presetLayout=QVBoxLayout() + # self.detailView.setFrameStyle(QFrame.Box) + self.presetLayout = QVBoxLayout() self.presetWidget.setLayout(self.presetLayout) self.detailName = QLabel('') @@ -117,8 +122,7 @@ def __init__(self, parent=None): self.detailDescription = TextViewer() self.presetLayout.addWidget(self.detailDescription) - - self.presetForm=QFormLayout() + self.presetForm = QFormLayout() self.presetForm.setRowWrapPolicy(QFormLayout.DontWrapRows); self.presetForm.setFieldGrowthPolicy(QFormLayout.AllNonFixedFieldsGrow); self.presetForm.setFormAlignment(Qt.AlignLeft | Qt.AlignTop); @@ -129,14 +133,14 @@ def __init__(self, parent=None): self.detailModule = QLabel('') self.presetForm.addRow('Module', self.detailModule) - #Options - self.detailOptionsLabel = QLabel('Options') + # Options + self.detailOptionsLabel = QLabel('Options') self.detailOptionsLabel.setStyleSheet("QLabel {height:25px;}") self.detailOptions = TextViewer() self.presetForm.addRow(self.detailOptionsLabel, self.detailOptions) # Columns - self.detailColumnsLabel = QLabel('Columns') + self.detailColumnsLabel = QLabel('Columns') self.detailColumnsLabel.setStyleSheet("QLabel {height:25px;}") self.detailColumns = TextViewer() self.presetForm.addRow(self.detailColumnsLabel, self.detailColumns) @@ -156,48 +160,50 @@ def __init__(self, parent=None): # Headers self.detailHeaders = QLabel('') self.presetForm.addRow('Header nodes', self.detailHeaders) - + # Buttons - buttons= QHBoxLayout() #QDialogButtonBox() + buttons = QHBoxLayout() + self.saveButton = QPushButton('New preset') self.saveButton.clicked.connect(self.newPreset) self.saveButton.setToolTip(wraptip("Create a new preset using the current tab and parameters")) - #buttons.addButton(self.saveButton,QDialogButtonBox.ActionRole) buttons.addWidget(self.saveButton) - self.overwriteButton = QPushButton('Edit preset') - self.overwriteButton.clicked.connect(self.overwritePreset) - self.overwriteButton.setToolTip(wraptip("Edit the selected preset.")) - #buttons.addButton(self.overwriteButton,QDialogButtonBox.ActionRole) + self.newPipelineButton = QPushButton('New pipeline') + self.newPipelineButton.clicked.connect(self.newPipeline) + self.newPipelineButton.setToolTip(wraptip("Create a new pipeline")) + buttons.addWidget(self.newPipelineButton) + + self.overwriteButton = QPushButton('Edit') + self.overwriteButton.clicked.connect(self.overwriteItem) + self.overwriteButton.setToolTip(wraptip("Edit the selected preset or pipeline.")) buttons.addWidget(self.overwriteButton) - self.deleteButton = QPushButton('Delete preset') + self.deleteButton = QPushButton('Delete') self.deleteButton.clicked.connect(self.deletePreset) - self.deleteButton.setToolTip(wraptip("Delete the selected preset. Default presets can not be deleted.")) - #buttons.addButton(self.deleteButton,QDialogButtonBox.ActionRole) + self.deleteButton.setToolTip( + wraptip("Delete the selected preset or pipeline. Default items can not be deleted.")) buttons.addWidget(self.deleteButton) - #layout.addWidget(buttons,1) - buttons.addStretch() - self.reloadButton=QPushButton('Reload') + self.reloadButton = QPushButton('Reload') self.reloadButton.clicked.connect(self.reloadPresets) self.reloadButton.setToolTip(wraptip("Reload all preset files.")) buttons.addWidget(self.reloadButton) - self.rejectButton=QPushButton('Cancel') + self.rejectButton = QPushButton('Cancel') self.rejectButton.clicked.connect(self.close) self.rejectButton.setToolTip(wraptip("Close the preset dialog.")) buttons.addWidget(self.rejectButton) - self.columnsButton=QPushButton('Add Columns') + self.columnsButton = QPushButton('Add Columns') self.columnsButton.setDefault(True) self.columnsButton.clicked.connect(self.addColumns) self.columnsButton.setToolTip(wraptip("Add the columns of the selected preset to the column setup.")) buttons.addWidget(self.columnsButton) - self.applyButton=QPushButton('Apply') + self.applyButton = QPushButton('Apply') self.applyButton.setDefault(True) self.applyButton.clicked.connect(self.applyPreset) self.applyButton.setToolTip(wraptip("Load the selected preset.")) @@ -205,23 +211,22 @@ def __init__(self, parent=None): layout.addLayout(buttons) - #status bar + # status bar self.statusbar = QStatusBar() - #self.folderLabel = QLabel("") + # self.folderLabel = QLabel("") self.folderButton = QPushButton("") self.folderButton.setFlat(True) self.folderButton.clicked.connect(self.statusBarClicked) - self.statusbar.insertWidget(0,self.folderButton) + self.statusbar.insertWidget(0, self.folderButton) layout.addWidget(self.statusbar) - - #self.presetFolder = os.path.join(os.path.dirname(self.mainWindow.settings.fileName()),'presets') - self.presetFolder = os.path.join(os.path.expanduser("~"),'Facepager','Presets') - self.presetFolderDefault = os.path.join(os.path.expanduser("~"),'Facepager','DefaultPresets') + # self.presetFolder = os.path.join(os.path.dirname(self.mainWindow.settings.fileName()),'presets') + self.presetFolder = os.path.join(os.path.expanduser("~"), 'Facepager', 'Presets') + self.presetFolderDefault = os.path.join(os.path.expanduser("~"), 'Facepager', 'DefaultPresets') self.folderButton.setText(self.presetFolder) self.presetsDownloaded = False - self.presetSuffix = ['.3_9.json','.3_10.json','.fp4.json'] + self.presetSuffix = ['.3_9.json', '.3_10.json', '.fp4.json'] self.lastSelected = None # Progress bar (sync with download thread by signals @@ -231,10 +236,11 @@ def __init__(self, parent=None): self.progressMax.connect(self.setProgressMax) self.progressStep.connect(self.setProgressStep) self.progressStop.connect(self.setProgressStop) -# if getattr(sys, 'frozen', False): -# self.defaultPresetFolder = os.path.join(os.path.dirname(sys.executable),'presets') -# elif __file__: -# self.defaultPresetFolder = os.path.join(os.path.dirname(__file__),'presets') + + # if getattr(sys, 'frozen', False): + # self.defaultPresetFolder = os.path.join(os.path.dirname(sys.executable),'presets') + # elif __file__: + # self.defaultPresetFolder = os.path.join(os.path.dirname(__file__),'presets') # Sycn progress bar with download thread @Slot() @@ -268,62 +274,66 @@ def statusBarClicked(self): if platform.system() == "Windows": webbrowser.open(self.presetFolder) elif platform.system() == "Darwin": - webbrowser.open('file:///'+self.presetFolder) + webbrowser.open('file:///' + self.presetFolder) else: - webbrowser.open('file:///'+self.presetFolder) - + webbrowser.open('file:///' + self.presetFolder) def currentChanged(self): - #hide - self.detailName.setText("") - self.detailModule.setText("") - self.detailDescription.setText("") - self.detailOptions.setText("") - self.detailColumns.setText("") - - self.presetView.hide() - self.categoryView.hide() + self.hideDetails() current = self.presetList.currentItem() - if current and current.isSelected(): - data = current.data(0,Qt.UserRole) - - # Single preset - if not data.get('iscategory',False): - self.lastSelected = os.path.join(data.get('folder',''),data.get('filename','')) - - self.detailName.setText(data.get('name')) - self.detailModule.setText(data.get('module')) - self.detailDescription.setText(data.get('description')+"\n") - self.detailOptions.setHtml(formatdict(data.get('options',[]))) - self.detailColumns.setText("\r\n".join(data.get('columns', []))) - self.detailSpeed.setText(str(data.get('speed',''))) - self.detailTimeout.setText(str(data.get('timeout', ''))) - self.detailMaxsize.setText(str(data.get('maxsize', ''))) - - #self.applyButton.setText("Apply") - self.presetView.show() - - # Category - else: -# self.pipelineName.setText(str(data.get('category'))) - -# self.pipelineWidget.clear() -# for i in range(current.childCount()): -# presetitem = current.child(i) -# preset = presetitem.data(0, Qt.UserRole) -# -# treeitem = QTreeWidgetItem(self.pipelineWidget) -# treeitem.setText(0,preset.get('name')) -# treeitem.setText(1, preset.get('module')) -# treeitem.setText(2, getDictValue(preset,'options.basepath')) -# treeitem.setText(3, getDictValue(preset,'options.resource')) -# # treeitem.setText(4, preset.get('description')) -# -# self.pipelineWidget.addTopLevelItem(treeitem) - - #self.applyButton.setText("Run pipeline") - self.categoryView.show() + if not current or not current.isSelected(): + return + + data = current.data(0, Qt.UserRole) + itemType = data.get('type', 'preset') + + # Single preset + if itemType == 'preset': + self.lastSelected = os.path.join(data.get('folder', ''), data.get('filename', '')) + + self.detailName.setText(data.get('name')) + self.detailModule.setText(data.get('module')) + self.detailDescription.setText(data.get('description', '') + "\n") + self.detailOptions.setHtml(formatdict(data.get('options', {}))) + self.detailColumns.setText("\r\n".join(data.get('columns', []))) + self.detailSpeed.setText(str(data.get('speed', ''))) + self.detailTimeout.setText(str(data.get('timeout', ''))) + self.detailMaxsize.setText(str(data.get('maxsize', ''))) + + self.applyButton.setText("Apply") + self.presetView.show() + + # Pipeline + elif itemType == 'pipeline': + self.pipelineName.setText("Pipeline {0}".format(str(data.get('name')))) + self.pipelineDescription.setText(data.get('description', '') + "\n") + + self.pipelineWidget.clear() + for i in range(current.childCount()): + presetitem = current.child(i) + preset = presetitem.data(0, Qt.UserRole) + + treeitem = QTreeWidgetItem(self.pipelineWidget) + treeitem.setText(0, preset.get('name')) + treeitem.setText(1, preset.get('module')) + treeitem.setText(2, getDictValue(preset, 'options.basepath')) + treeitem.setText(3, getDictValue(preset, 'options.resource')) + # treeitem.setText(4, preset.get('description')) + + self.pipelineWidget.addTopLevelItem(treeitem) + + self.applyButton.setText("Run pipeline") + self.pipelineView.show() + + def selectItem(self, item, step=None): + self.presetList.setCurrentItem(item) + if step is not None: + stepRoot = self.pipelineWidget.invisibleRootItem() + if stepRoot and stepRoot.childCount() > step: + stepItem = stepRoot.child(step) + self.pipelineWidget.clearSelection() + self.pipelineWidget.setItemSelected(stepItem, True) def showPresets(self): self.clear() @@ -332,14 +342,109 @@ def showPresets(self): self.progressShow.emit() - self.initPresets() + self.loadPresets() self.raise_() + def addCategoryItem(self, data, default): + """ + Add a category item if it does not exist, otherwise return the existing item + + :param data: A dict with the keys module and category + :return: A PresetWidgetItem() with the iscategory property set to True + """ + category = data.get('category', 'No category') + + if not category in self.categoryNodes: + categoryItem = PresetWidgetItem() + categoryItem.setText(0, category) + + ft = categoryItem.font(0) + ft.setWeight(QFont.Bold) + categoryItem.setFont(0, ft) - def addPresetItem(self,folder,filename,default=False,online=False): + categoryItem.setData( + 0, Qt.UserRole, + {'iscategory': True, 'name': category, 'category': category, 'type': 'category'} + ) + + self.presetList.addTopLevelItem(categoryItem) + self.categoryNodes[category] = categoryItem + + else: + categoryItem = self.categoryNodes[category] + + return categoryItem + + def addPresetItem(self, parentItem, data, default): + data['type'] = 'preset' + data['caption'] = data.get('name') + if default: + data['caption'] = data['caption'] + " *" + + newItem = PresetWidgetItem() + newItem.setText(0, data['caption']) + newItem.setData(0, Qt.UserRole, data) + if default: + newItem.setForeground(0, QBrush(QColor("darkblue"))) + + parentItem.addChild(newItem) + return newItem + + def addPipelineDummyItem(self, parentItem, data, default): + pipelineData = {} + pipelineData['category'] = data.get('category', 'No category') + pipelineData['name'] = data.get('pipeline', '') + pipelineData['caption'] = data.get('pipeline', '') + + return self.addPipelineItem(parentItem, pipelineData, default) + + def addPipelineItem(self, parentItem, pipelineData, default): + pipelineData['category'] = pipelineData.get('category', 'No category') + pipelineData['caption'] = pipelineData.get('name', '') + + if default: + pipelineData['caption'] = pipelineData['caption'] + " *" + + if not pipelineData['category'] in self.pipelineNodes.keys(): + self.pipelineNodes[pipelineData['category']] = {} + pipelineCategory = self.pipelineNodes[pipelineData['category']] + + if not pipelineData['caption'] in pipelineCategory.keys(): + pipelineData['type'] = 'pipeline' + pipelineItem = PresetWidgetItem() + + ft = pipelineItem.font(0) + ft.setWeight(QFont.Bold) + pipelineItem.setFont(0, ft) + + pipelineItem.setText(0, pipelineData['caption']) + if default: + pipelineItem.setForeground(0, QBrush(QColor("darkblue"))) + pipelineItem.setData(0, Qt.UserRole, pipelineData) + parentItem.addChild(pipelineItem) + pipelineCategory[pipelineData['caption']] = pipelineItem + else: + pipelineItem = pipelineCategory[pipelineData['caption']] + + # Only overwrite real pipelines + if pipelineData.get('type', 'preset') == 'pipeline': + pipelineItem.setData(0, Qt.UserRole, pipelineData) + + return pipelineItem + + def addItem(self, folder, filename, default=False, online=False): + """ + Add a new item to the preset tree widget + + :param folder: The folder with the preset or pipeline file (default or custom presets folder) + :param filename: The preset or pipeline file name + :param default: Whether this is a preset or pipeline shipped with Facepager + :param online: Whether folder and filename together are a download URL + :return: + """ try: if online: - data= requests.get(folder+filename).json() + data = requests.get(folder + filename).json() else: with open(os.path.join(folder, filename), 'r', encoding="utf-8") as input: data = json.load(input) @@ -348,76 +453,84 @@ def addPresetItem(self,folder,filename,default=False,online=False): data['folder'] = folder data['default'] = default data['online'] = online + data['type'] = data.get('type', 'preset') - if data.get('type','preset') == 'pipeline': - data['category'] = data.get('category', 'noname') - if not data['category'] in self.categoryNodes: - categoryItem = PresetWidgetItem() - categoryItem.setText(0, data['category']) + # First level: category + data['category'] = data.get('category', 'No category') + parentItem = self.addCategoryItem(data, default) - ft = categoryItem.font(0) - ft.setWeight(QFont.Bold) - categoryItem.setFont(0, ft) + # Pipelines on second level + if data['type'] == 'pipeline': + newItem = self.addPipelineItem(parentItem, data, default) + # Presets on second (standalone) or third (in a pipeline) level + else: + pipelineName = data.get('pipeline', '') + if (pipelineName != '') and (pipelineName is not None): + parentItem = self.addPipelineDummyItem(parentItem, data, default) + newItem = self.addPresetItem(parentItem, data, default) - self.presetList.addTopLevelItem(categoryItem) - else: - categoryItem = self.categoryNodes[data['category']] + QApplication.processEvents() + return newItem - data['iscategory'] = True - categoryItem.setData(0, Qt.UserRole,data) + except Exception as e: + QMessageBox.information(self, "Facepager", "Error loading preset:" + str(e)) + return None - else: - data['caption'] = data.get('name') - if default: - data['caption'] = data['caption'] +" *" - - data['category'] = data.get('category','') - if (data['category'] == ''): - if (data.get('module') in ['Generic','Files']): - try: - data['category'] = data.get('module') + " ("+urlparse(data['options']['basepath']).netloc+")" - except: - data['category'] = data.get('module') - else: - data['category'] = data.get('module') + def updateItem(self, item, folder, filename): + itemData = item.data(0, Qt.UserRole) + reloadFiles = [filename] - if not data['category'] in self.categoryNodes: - categoryItem = PresetWidgetItem() - categoryItem.setText(0,data['category']) + # Update children + if (itemData.get('type') == 'pipeline') and (item.childCount() > 0): + # Load the first JSON file + with open(os.path.join(folder, filename), 'r') as file: + pipelineData = json.load(file) + category = pipelineData['category'] + pipeline = pipelineData['name'] - ft = categoryItem.font(0) - ft.setWeight(QFont.Bold) - categoryItem.setFont(0,ft) + while (item.childCount() > 0): + childItem = item.child(0) + childData = childItem.data(0, Qt.UserRole) + childFilename = childData.get('filename') - categoryItem.setData(0,Qt.UserRole,{'iscategory':True,'name':data['module'],'category':data['category']}) + with open(os.path.join(folder, childFilename), 'r') as file: + childData = json.load(file) - self.presetList.addTopLevelItem(categoryItem) - self.categoryNodes[data['category']] = categoryItem + childData['category'] = category + childData['pipeline'] = pipeline - else: - categoryItem = self.categoryNodes[data['category']] + with open(os.path.join(folder, childFilename), 'w') as file: + json.dump(childData, file,indent=2, separators=(',', ': ')) - newItem = PresetWidgetItem() - newItem.setText(0,data['caption']) - newItem.setData(0,Qt.UserRole,data) - if default: - newItem.setForeground(0,QBrush(QColor("darkblue"))) - categoryItem.addChild(newItem) + reloadFiles.append(childFilename) + item.removeChild(childItem) - #self.presetList.setCurrentItem(newItem,0) - QApplication.processEvents() + # Remove pipeline item from index + if itemData.get('category') in self.pipelineNodes.keys(): + if itemData.get('caption') in self.pipelineNodes[itemData.get('category')].keys(): + del self.pipelineNodes[itemData.get('category')][itemData.get('caption')] - return newItem + item.parent().removeChild(item) - except Exception as e: - QMessageBox.information(self,"Facepager","Error loading preset:"+str(e)) - return None + for reloadFile in reloadFiles: + self.addItem(folder, reloadFile) + + def hideDetails(self): + self.detailName.setText("") + self.detailModule.setText("") + self.detailDescription.setText("") + self.detailOptions.setText("") + self.detailColumns.setText("") + + self.presetView.hide() + self.pipelineDescription.setText("") + self.pipelineView.hide() def clear(self): self.presetList.clear() self.presetView.hide() - self.categoryView.hide() + self.pipelineView.hide() self.loadingIndicator.show() def checkDefaultFiles(self): @@ -436,41 +549,70 @@ def downloadDefaultFiles(self, silent=False): if not silent: self.progressShow.emit() + # Create folder + if not os.path.exists(self.presetFolderDefault): + os.makedirs(self.presetFolderDefault) + + # Lock file handling + lockedShas = {} + newShas = {} + lock_file_path = os.path.join(self.presetFolderDefault, 'presets.lock') + if not os.path.exists(lock_file_path): + open(lock_file_path, 'w').close() + else: + with open(lock_file_path, 'r') as file: + lockedShas = {line.strip().split(' ', 1)[1]: line.strip().split(' ', 1)[0] for line in + file.readlines()} + # Create temporary download folder tmp = TemporaryDirectory(suffix='FacepagerDefaultPresets') try: - #Download - files = requests.get("https://api.github.com/repos/strohne/Facepager/contents/presets").json() - files = [f['path'] for f in files if f['path'].endswith(tuple(self.presetSuffix))] + # Download + files = requests.get(settings.get("presetListUrl")).json() + files = [f for f in files if f['path'].endswith(tuple(self.presetSuffix))] self.progressMax.emit(len(files)) - for filename in files: + for fileInfo in files: if self.progress.wasCanceled: raise Exception(f"Downloading default presets was canceled by you.") - response = requests.get("https://raw.githubusercontent.com/strohne/Facepager/master/"+filename) - if response.status_code != 200: - raise Exception(f"GitHub is not available (status code {response.status_code})") - with open(os.path.join(tmp.name, os.path.basename(filename)), 'wb') as f: - f.write(response.content) - self.progressStep.emit() + sha = fileInfo['sha'] + filename = fileInfo['path'] + basename = os.path.basename(filename) + + tmpPath = os.path.join(tmp.name, basename) + targetPath = os.path.join(self.presetFolderDefault, basename) - #Create folder - if not os.path.exists(self.presetFolderDefault): - os.makedirs(self.presetFolderDefault) + if sha not in lockedShas.values() or lockedShas.get(basename) != sha: + response = requests.get(settings.get("presetFileUrl") + filename) + if response.status_code != 200: + raise Exception(f"GitHub is not available (status code {response.status_code})") + with open(tmpPath, 'wb') as f: + f.write(response.content) + newShas[basename] = sha + elif os.path.exists(targetPath): + shutil.copyfile(targetPath, tmpPath) + newShas[basename] = sha + + self.progressStep.emit() - #Clear folder + # Clear folder for filename in os.listdir(self.presetFolderDefault): - os.remove(os.path.join(self.presetFolderDefault,filename)) + os.remove(os.path.join(self.presetFolderDefault, filename)) # Move files from tempfolder for filename in os.listdir(tmp.name): - shutil.move(os.path.join(tmp.name,filename), self.presetFolderDefault) + shutil.move(os.path.join(tmp.name, filename), self.presetFolderDefault) + + # Update the lock file to reflect actual directory contents + with open(lock_file_path, 'w') as file: + for filename, sha in newShas.items(): + file.write(f"{sha} {filename}\n") self.logmessage.emit("Default presets downloaded from GitHub.") except Exception as e: if not silent: - QMessageBox.information(self,"Facepager","Error downloading default presets:"+str(e)) + QMessageBox.information(self, "Facepager", "Error downloading default presets:" + str(e)) self.logmessage.emit("Error downloading default presets:" + str(e)) return False else: @@ -483,16 +625,16 @@ def downloadDefaultFiles(self, silent=False): def reloadPresets(self): self.presetsDownloaded = False self.downloadDefaultFiles() - self.initPresets() + self.loadPresets() - def initPresets(self): + def loadPresets(self): self.loadingIndicator.show() - #self.defaultPresetFolder self.categoryNodes = {} + self.pipelineNodes = {} self.presetList.clear() self.presetView.hide() - self.categoryView.hide() + self.pipelineView.hide() selectitem = None @@ -502,20 +644,22 @@ def initPresets(self): if os.path.exists(self.presetFolderDefault): files = [f for f in os.listdir(self.presetFolderDefault) if f.endswith(tuple(self.presetSuffix))] for filename in files: - newitem = self.addPresetItem(self.presetFolderDefault,filename,True) - if self.lastSelected is not None and (self.lastSelected == os.path.join(self.presetFolderDefault,filename)): + newitem = self.addItem(self.presetFolderDefault, filename, True) + if self.lastSelected is not None and ( + self.lastSelected == os.path.join(self.presetFolderDefault, filename)): selectitem = newitem if os.path.exists(self.presetFolder): files = [f for f in os.listdir(self.presetFolder) if f.endswith(tuple(self.presetSuffix))] for filename in files: - newitem = self.addPresetItem(self.presetFolder,filename) - if self.lastSelected is not None and (self.lastSelected == str(os.path.join(self.presetFolder,filename))): + newitem = self.addItem(self.presetFolder, filename) + if self.lastSelected is not None and ( + self.lastSelected == str(os.path.join(self.presetFolder, filename))): selectitem = newitem - #self.presetList.expandAll() + # self.presetList.expandAll() self.presetList.setFocus() - self.presetList.sortItems(0,Qt.AscendingOrder) + self.presetList.sortItems(0, Qt.AscendingOrder) selectitem = self.presetList.topLevelItem(0) if selectitem is None else selectitem self.presetList.setCurrentItem(selectitem) @@ -530,26 +674,45 @@ def getCategories(self): for i in range(root.childCount()): item = root.child(i) data = item.data(0, Qt.UserRole) - categories.append(data.get('category', '')) + categories.append(data.get('category', 'No category')) return categories - def startPipeline(self): + def getPipelines(self): + pipelines = [''] + + root = self.presetList.invisibleRootItem() + for i in range(root.childCount()): + categoryItem = root.child(i) + for j in range(categoryItem.childCount()): + item = categoryItem.child(j) + itemData = item.data(0, Qt.UserRole) + itemName = itemData.get('name', '') if itemData else '' + itemType = itemData.get('type') if itemData else '' + + if itemType == 'pipeline' and not itemName in pipelines: + pipelines.append(itemName) + + return pipelines + + def startPipeline(self, pipelineData): if not self.presetList.currentItem(): return False - # Get category item - root_item = self.presetList.currentItem() - root_data = root_item.data(0, Qt.UserRole) - if not root_data.get('iscategory', False): - root_item = root_item.parent() - root_data = root_item.data(0, Qt.UserRole) + # Get pipeline item + pipelineItem = self.presetList.currentItem() + root_data = pipelineItem.data(0, Qt.UserRole) + if root_data.get('type', '') != 'pipeline': + return False # Create pipeline - pipeline = [] + pipeline = { + 'item': pipelineItem, + 'presets': [] + } - for i in range(root_item.childCount()): - item = root_item.child(i) + for i in range(pipelineItem.childCount()): + item = pipelineItem.child(i) preset = item.data(0, Qt.UserRole) module = self.mainWindow.getModule(preset.get('module', None)) @@ -557,23 +720,22 @@ def startPipeline(self): options.update(preset.get('options', {})) preset['options'] = options.copy() preset['item'] = item + preset['step'] = i - pipeline.append(preset) + pipeline['presets'].append(preset) # Process pipeline return self.mainWindow.apiActions.queryPipeline(pipeline) - #self.close() - def applyPreset(self): if not self.presetList.currentItem(): return False - data = self.presetList.currentItem().data(0,Qt.UserRole) - if data.get('iscategory',False): - return False - #self.startPipeline() - else: + data = self.presetList.currentItem().data(0, Qt.UserRole) + itemType = data.get('type', 'preset') + if itemType == 'pipeline': + self.startPipeline(data) + elif itemType == 'preset': self.mainWindow.apiActions.applySettings(data) self.close() @@ -582,17 +744,16 @@ def addColumns(self): if not self.presetList.currentItem(): return False - data = self.presetList.currentItem().data(0,Qt.UserRole) - if not data.get('iscategory',False): + data = self.presetList.currentItem().data(0, Qt.UserRole) + if data.get('type', 'preset') == 'preset': self.mainWindow.apiActions.addColumns(data) - - def uniqueFilename(self,name): - filename = re.sub('[^a-zA-Z0-9_-]+', '_', name )+self.presetSuffix[-1] + def uniqueFilename(self, name): + filename = re.sub('[^a-zA-Z0-9_-]+', '_', name) + self.presetSuffix[-1] i = 1 while os.path.exists(os.path.join(self.presetFolder, filename)) and i < 10000: - filename = re.sub('[^a-zA-Z0-9_-]+', '_', name )+"-"+str(i)+self.presetSuffix[-1] - i+=1 + filename = re.sub('[^a-zA-Z0-9_-]+', '_', name) + "-" + str(i) + self.presetSuffix[-1] + i += 1 if os.path.exists(os.path.join(self.presetFolder, filename)): raise Exception('Could not find unique filename') @@ -601,98 +762,138 @@ def uniqueFilename(self,name): def deletePreset(self): if not self.presetList.currentItem(): return False - data = self.presetList.currentItem().data(0,Qt.UserRole) - if data.get('default',False): - QMessageBox.information(self,"Facepager","Cannot delete default presets.") + data = self.presetList.currentItem().data(0, Qt.UserRole) + if data.get('default', False): + QMessageBox.information(self, "Facepager", "Cannot delete default presets.") + return False + + if self.presetList.currentItem().childCount() > 0: + QMessageBox.information(self, "Facepager", "Delete the child nodes first.") return False - if data.get('iscategory',False): + if data.get('type', 'preset') == 'category': return False - reply = QMessageBox.question(self, 'Delete Preset',"Are you sure to delete the preset \"{0}\"?".format(data.get('name','')), QMessageBox.Yes | QMessageBox.No, QMessageBox.No) + reply = QMessageBox.question( + self, + 'Delete Preset', + "Are you sure to delete the {0} \"{1}\"?".format( + data.get('type', ''), + data.get('name', '') + ), + QMessageBox.Yes | QMessageBox.No, QMessageBox.No + ) + if reply != QMessageBox.Yes: return False - os.remove(os.path.join(self.presetFolder, data.get('filename'))) - self.initPresets() + if (data.get('filename', '') != ''): + filePath = os.path.join(self.presetFolder, data.get('filename')) + if os.path.exists(filePath): + os.remove(filePath) + + self.loadPresets() - def editPreset(self,data = None): - dialog=QDialog(self.mainWindow) + def editItem(self, data=None): + dialog = QDialog(self.mainWindow) self.currentData = data if data is not None else {} - self.currentFilename = self.currentData.get('filename',None) + self.currentFilename = self.currentData.get('filename', None) + itemType = self.currentData.get('type', 'preset') if self.currentFilename is None: - dialog.setWindowTitle("New Preset") + dialog.setWindowTitle("New {0}".format(itemType)) else: - dialog.setWindowTitle("Edit selected preset") + dialog.setWindowTitle("Edit selected {0}".format(itemType)) - layout=QVBoxLayout() - label=QLabel("Name") + layout = QVBoxLayout() + label = QLabel("Name") layout.addWidget(label) - name=QLineEdit() - name.setText(self.currentData.get('name','')) - layout.addWidget(name,0) + name = QLineEdit() + name.setText(self.currentData.get('name', '')) + layout.addWidget(name, 0) - label=QLabel("Category") + label = QLabel("Category") layout.addWidget(label) - category= QComboBox(self) - category.addItems(self.getCategories()) - category.setEditable(True) - - category.setCurrentText(self.currentData.get('category','')) - layout.addWidget(category,0) - + categoryWidget = QComboBox(self) + categoryWidget.addItems(self.getCategories()) + categoryWidget.setEditable(True) + categoryWidget.setCurrentText(self.currentData.get('category', 'No category')) + layout.addWidget(categoryWidget, 0) + + if itemType == 'preset': + label = QLabel("Pipeline") + layout.addWidget(label) + pipelineWidget = QComboBox(self) + pipelineWidget.addItems(self.getPipelines()) + pipelineWidget.setCurrentIndex(pipelineWidget.findText(self.currentData.get('pipeline', ''))) + layout.addWidget(pipelineWidget, 0) + else: + pipelineWidget = None - label=QLabel("Description") + label = QLabel("Description") layout.addWidget(label) - description=QTextEdit() + description = QTextEdit() description.setMinimumWidth(500) - description.acceptRichText=False - description.setPlainText(self.currentData.get('description','')) + description.acceptRichText = False + description.setPlainText(self.currentData.get('description', '')) description.setFocus() - layout.addWidget(description,1) - + layout.addWidget(description, 1) - overwriteLayout =QHBoxLayout() - self.overwriteCheckbox = QCheckBox(self) - self.overwriteCheckbox.setCheckState(Qt.Unchecked) - overwriteLayout.addWidget(self.overwriteCheckbox) - label=QLabel("Overwrite parameters with current settings") - overwriteLayout.addWidget(label) - overwriteLayout.addStretch() + if itemType == 'preset': + overwriteLayout = QHBoxLayout() + overwriteCheckbox = QCheckBox(self) + overwriteCheckbox.setCheckState(Qt.Unchecked) + overwriteLayout.addWidget(overwriteCheckbox) + label = QLabel("Overwrite parameters with current settings") + overwriteLayout.addWidget(label) + overwriteLayout.addStretch() - if self.currentFilename is not None: - layout.addLayout(overwriteLayout) + if self.currentFilename is not None: + layout.addLayout(overwriteLayout) + else: + overwriteCheckbox = None - buttons=QDialogButtonBox(QDialogButtonBox.Ok|QDialogButtonBox.Cancel) - layout.addWidget(buttons,0) + buttons = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) + layout.addWidget(buttons, 0) dialog.setLayout(layout) def save(): + if pipelineWidget: + pipelineName = pipelineWidget.currentText() + if pipelineName == '': + pipelineName = None + else: + pipelineName = None + data_meta = { - 'name':name.text(), - 'category':category.currentText(), - 'description':description.toPlainText() + 'name': name.text(), + 'category': categoryWidget.currentText(), + 'pipeline': pipelineName, + 'description': description.toPlainText() } - - data_settings = self.mainWindow.apiActions.getPresetOptions() self.currentData.update(data_meta) - if self.currentFilename is None: - self.currentData.update(data_settings) - - elif self.overwriteCheckbox.isChecked(): - reply = QMessageBox.question(self, 'Overwrite Preset',"Are you sure to overwrite the selected preset \"{0}\" with the current settings?".format(data.get('name','')), QMessageBox.Yes | QMessageBox.No, QMessageBox.No) - if reply != QMessageBox.Yes: - dialog.close() - self.currentFilename = None - return False - else: + if itemType == 'preset': + data_settings = self.mainWindow.apiActions.getPresetOptions() + if self.currentFilename is None: self.currentData.update(data_settings) + elif overwriteCheckbox is not None and overwriteCheckbox.isChecked(): + reply = QMessageBox.question(self, 'Overwrite Preset', + "Are you sure to overwrite the selected preset \"{0}\" with the current settings?".format( + data.get('name', '')), QMessageBox.Yes | QMessageBox.No, + QMessageBox.No) + if reply != QMessageBox.Yes: + dialog.close() + self.currentFilename = None + return False + else: + self.currentData.update(data_settings) + # Sanitize and reorder - keys = ['name', 'category', 'description', 'module', 'options', 'speed', 'saveheaders','timeout','maxsize','columns'] + keys = ['name', 'category', 'type', 'pipeline', 'description', 'module', 'options', 'speed', 'saveheaders', + 'timeout', 'maxsize', 'columns'] self.currentData = {k: self.currentData.get(k, None) for k in keys} # Create folder @@ -706,11 +907,16 @@ def save(): os.remove(filepath) # Save new file - catname = category.currentText() if category.currentText() != "" else self.mainWindow.RequestTabs.currentWidget().name - self.currentFilename = self.uniqueFilename(catname+"-"+name.text()) + categoryName = self.currentData.get('category', 'No category') + pipelineName = self.currentData.get('pipeline', '') + newFilename = categoryName + '-' + if pipelineName is not None and pipelineName != '': + newFilename = newFilename + pipelineName + '-' + newFilename = newFilename + name.text() + self.currentFilename = self.uniqueFilename(newFilename) - with open(os.path.join(self.presetFolder,self.currentFilename), 'w') as outfile: - json.dump(self.currentData, outfile,indent=2, separators=(',', ': ')) + with open(os.path.join(self.presetFolder, self.currentFilename), 'w') as outfile: + json.dump(self.currentData, outfile, indent=2, separators=(',', ': ')) dialog.close() return True @@ -720,62 +926,55 @@ def close(): self.currentFilename = None return False - #connect the nested functions above to the dialog-buttons + # connect the nested functions above to the dialog-buttons buttons.accepted.connect(save) buttons.rejected.connect(close) dialog.exec() return self.currentFilename - def newPreset(self): - filename = self.editPreset() + filename = self.editItem({'type': 'preset'}) if filename is not None: - newitem = self.addPresetItem(self.presetFolder,filename) - self.presetList.sortItems(0,Qt.AscendingOrder) - self.presetList.setCurrentItem(newitem,0) + newitem = self.addItem(self.presetFolder, filename) + self.presetList.sortItems(0, Qt.AscendingOrder) + self.presetList.setCurrentItem(newitem, 0) + def newPipeline(self): + filename = self.editItem({'type': 'pipeline'}) + if filename is not None: + newitem = self.addItem(self.presetFolder, filename) + self.presetList.sortItems(0, Qt.AscendingOrder) + self.presetList.setCurrentItem(newitem, 0) - def overwritePreset(self): + def overwriteItem(self): if not self.presetList.currentItem(): return False item = self.presetList.currentItem() - data = item.data(0,Qt.UserRole) + data = item.data(0, Qt.UserRole) - if data.get('default',False): - QMessageBox.information(self,"Facepager","Cannot edit default presets.") + if data.get('default', False): + QMessageBox.information(self, "Facepager", "Cannot edit default presets.") return False - if data.get('iscategory',False): + if data.get('type', 'preset') == 'category': return False - filename = self.editPreset(data) + itemFilename = self.editItem(data) - if filename is not None: - item.parent().removeChild(item) - item = self.addPresetItem(self.presetFolder,filename) + # Reload + if itemFilename is not None: + self.updateItem(item, self.presetFolder, itemFilename) + self.presetList.sortItems(0, Qt.AscendingOrder) + self.presetList.setCurrentItem(item, 0) - self.presetList.sortItems(0,Qt.AscendingOrder) - self.presetList.setCurrentItem(item,0) class PresetWidgetItem(QTreeWidgetItem): def __lt__(self, other): - data1 = self.data(0,Qt.UserRole) - data2 = other.data(0,Qt.UserRole) - - if data1.get('iscategory') and data2.get('iscategory'): - #order = ['Facebook','YouTube','Twitter','Twitter Streaming','Amazon','Files','Generic'] - # if data1.get('name','') in order and data2.get('name','') in order: - # if data1.get('name','') == data2.get('name',''): - # return data1.get('category','') < data2.get('category','') - # else: - # return order.index(data1.get('name','')) < order.index(data2.get('name','')) - # - # elif (data1.get('name','') in order) != (data2.get('name','') in order): - # return data1.get('name','') in order - # else: - return data1.get('category','').lower() < data2.get('category','').lower() - elif data1.get('default',False) != data2.get('default',False): - return data1.get('default',False) + data1 = self.data(0, Qt.UserRole) + data2 = other.data(0, Qt.UserRole) + + if data1.get('default', False) != data2.get('default', False): + return data1.get('default', False) else: - return data1.get('name','').lower() < data2.get('name','').lower() + return data1.get('name', '').lower() < data2.get('name', '').lower() diff --git a/src/settings.py b/src/settings.py new file mode 100644 index 00000000..624e8f03 --- /dev/null +++ b/src/settings.py @@ -0,0 +1,6 @@ +settings = { + "version" : "4.6", + "presetListUrl" : "https://api.github.com/repos/strohne/Facepager/contents/presets", + "presetFileUrl": "https://raw.githubusercontent.com/strohne/Facepager/master/", + "userAgent" : 'FACEPAGERBOT/4.6 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.6' +} \ No newline at end of file diff --git a/src/utilities.py b/src/utilities.py index a9f31425..5f6f7a79 100644 --- a/src/utilities.py +++ b/src/utilities.py @@ -14,6 +14,7 @@ from xmljson import BadgerFish import io import pyjsparser +import rdflib def getResourceFolder(): if getattr(sys, 'frozen', False) and (platform.system() != 'Darwin'): @@ -413,6 +414,7 @@ def toDictListTuple(data, key=''): def getDictValue(data, multikey, dump=True, default=''): """Extract value from dict + :param data: :param multikey: :param dump: @@ -643,6 +645,17 @@ def elementToJson(element, context=True): return out +def extractTriples(data, format='turtle'): + g = rdflib.Graph() + g.parse(data=data, format=format) + + triples = [{ + 'subject': str(triple[0]), + 'predicate': str(triple[1]), + 'object': str(triple[2]) + } for triple in g] + + return triples def extractLinks(data,baseurl,parseurl=True): links = [] @@ -663,6 +676,8 @@ def extractLinks(data,baseurl,parseurl=True): return links, base + + urlcache = {} def extractURLparts(url_absolut,prefix="url_",usecache=False): cachekey = prefix + url_absolut @@ -842,7 +857,10 @@ def getdictvalues(data, parentkeys = []): out.extend([child]) return out - return "\n".join(getdictvalues(data)) + if data is None: + return "None" + else: + return "\n".join(getdictvalues(data)) diff --git a/src/widgets/paramedit.py b/src/widgets/paramedit.py index 067838d5..e062ac32 100644 --- a/src/widgets/paramedit.py +++ b/src/widgets/paramedit.py @@ -310,59 +310,99 @@ def verticalResizeTableViewToContents(self): self.setMinimumHeight(rowTotalHeight) self.setMaximumHeight(rowTotalHeight) -class ValueEdit(QWidget): - def __init__(self,parent): - super(ValueEdit, self).__init__(parent) - self.mainLayout = QHBoxLayout(self) - self.mainLayout.setContentsMargins(5,0,0,0) - self.setLayout(self.mainLayout) +class EditValueDialog(QDialog): + def __init__(self, parent=None): + super(EditValueDialog, self).__init__(parent, Qt.WindowSystemMenuHint | Qt.WindowTitleHint | Qt.WindowStaysOnTopHint) - self.comboBox = QComboBox(self) - self.comboBox.setEditable(True) + # The original widget + self.editWidget = None + self.setWindowTitle("Edit value") + self.resize(600, 600) - self.actionEditValue = QAction('...',self) - self.actionEditValue.setText('..') - self.actionEditValue.triggered.connect(self.editValue) + layout = QVBoxLayout(self) + self.setLayout(layout) - self.button =QToolButton(self) - self.button.setToolButtonStyle(Qt.ToolButtonTextOnly) - self.button.setDefaultAction(self.actionEditValue) + self.input = QPlainTextEdit(self) + self.input.setMinimumWidth(50) + layout.addWidget(self.input) + + # buttons + buttons = QHBoxLayout() + + buttons.addStretch() - self.mainLayout.addWidget(self.comboBox,2) - self.mainLayout.addWidget(self.button,0) + self.cancelButton = QPushButton('Cancel') + self.cancelButton.clicked.connect(self.reject) + self.cancelButton.setToolTip("Close the window without applying parameteres.") + buttons.addWidget(self.cancelButton) - def editValue(self): - dialog = QDialog(self,Qt.WindowSystemMenuHint | Qt.WindowTitleHint) - dialog.setWindowTitle("Edit value") - layout = QVBoxLayout() + self.closeButton = QPushButton('Close') + self.closeButton.clicked.connect(self.setValue) + self.closeButton.setToolTip("Apply parameters and close the window.") + buttons.addWidget(self.closeButton) + self.applyButton = QPushButton('Apply') + self.applyButton.setDefault(True) + self.applyButton.clicked.connect(self.applyValue) + self.applyButton.setToolTip("Apply parameters to module.") + buttons.addWidget(self.applyButton) + layout.addLayout(buttons) - input = QPlainTextEdit() - input.setMinimumWidth(50) - input.setPlainText(self.comboBox.currentText()) - #input.LineWrapMode = QPlainTextEdit.NoWrap - #input.acceptRichText=False - input.setFocus() - layout.addWidget(input) - buttons = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) - layout.addWidget(buttons) + def show(self, editWidget): + self.editWidget = editWidget - dialog.setLayout(layout) + if isinstance(self.editWidget, QComboBox): + self.input.setPlainText(self.editWidget.currentText()) + elif isinstance(self.editWidget, QPlainTextEdit): + self.input.setPlainText(self.editWidget.toPlainText()) - def setValue(): - value = input.toPlainText() #input.toPlainText().splitlines() - self.comboBox.setEditText(value) + self.input.setFocus() - dialog.close() + super(EditValueDialog, self).show() - def close(): - dialog.close() + def setValue(self): + value = self.input.toPlainText() + if isinstance(self.editWidget, QComboBox): + self.editWidget.setEditText(value) + elif isinstance(self.editWidget, QPlainTextEdit): + self.editWidget.setPlainText(value) + self.accept() + + def applyValue(self): + value = self.input.toPlainText() + if isinstance(self.editWidget, QComboBox): + self.editWidget.setEditText(value) + elif isinstance(self.editWidget, QPlainTextEdit): + self.editWidget.setPlainText(value) + +class ValueEdit(QWidget): + def __init__(self, parent=None): + super(ValueEdit, self).__init__(parent) + + self.moreEditor = None + + self.mainLayout = QHBoxLayout(self) + self.mainLayout.setContentsMargins(5, 0, 0, 0) + self.setLayout(self.mainLayout) + + self.comboBox = QComboBox(self) + self.comboBox.setEditable(True) + + self.actionEditValue = QAction('...', self) + self.actionEditValue.setText('..') + self.actionEditValue.triggered.connect(self.openEditDialog) + + self.button = QToolButton(self) + self.button.setToolButtonStyle(Qt.ToolButtonTextOnly) + self.button.setDefaultAction(self.actionEditValue) - #connect the nested functions above to the dialog-buttons - buttons.accepted.connect(setValue) - buttons.rejected.connect(close) - dialog.exec_() + self.mainLayout.addWidget(self.comboBox, 2) + self.mainLayout.addWidget(self.button, 0) + def openEditDialog(self): + if self.moreEditor is None: + self.moreEditor = EditValueDialog(self) + self.moreEditor.show(self.comboBox) diff --git a/src/widgets/textviewer.py b/src/widgets/textviewer.py index a41903df..132b0db7 100644 --- a/src/widgets/textviewer.py +++ b/src/widgets/textviewer.py @@ -22,7 +22,7 @@ def sizeChanged(self): def setText(self,text): text = '' if text is None else text - text = self.autoLinkText(text) + #text = self.autoLinkText(text) # auto br prevents markdown parsing # todo: make presets markdown compatible @@ -34,14 +34,18 @@ def setText(self,text): self.setHtml(text) def autoBrText(self,html): - return html.replace('\n', '
') + return html.replace('\n', '
') def autoLinkText(self,html): - # match all the urls - # this returns a tuple with two groups - # if the url is part of an existing link, the second element - # in the tuple will be "> or - # if not, the second element will be an empty string + """ + @deprecated + + match all the urls + this returns a tuple with two groups + if the url is part of an existing link, the second element + in the tuple will be "> or + if not, the second element will be an empty string + """ urlre = re.compile("(\(?https?://[-A-Za-z0-9+&@#/%?=~_()|!:,.;]*[-A-Za-z0-9+&@#/%=~_()|])(\">|)?") urls = urlre.findall(html) clean_urls = []