Merge remote-tracking branch 'origin/staging'

client/package.json

@@ -30,6 +30,7 @@
     "highcharts": "^11.3.0",
     "highcharts-react-official": "^3.2.1",
     "leaflet": "^1.9.4",
+    "ml-regression-simple-linear": "^3.0.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-intersection-observer": "^9.5.3",

client/src/api/networks/network/communities.ts

@@ -3,13 +3,14 @@ import louvain from "graphology-communities-louvain"
 import seedrandom from "seedrandom"
 import { arrayPush, labelClean } from "../_utils/functions"
 import { networkSearchHits, networkSearchAggs } from "../search/search"
-import { ElasticAggregations, ElasticHits, NetworkCommunities, NetworkFilters } from "../../../types/network"
+import { ElasticHits, NetworkCommunities, NetworkFilters } from "../../../types/network"
 import { openAiLabeledClusters } from "./mistralai"
 import { COLORS } from "../_utils/constants"
 import { GetColorName } from "hex-color-to-color-name"
 import { configGetItemUrl } from "./config"
 import { CONFIG } from "./config"
 import { nodeGetId } from "./network"
+import { ElasticAggregations } from "../../../types/commons"
 
 const CURRENT_YEAR = new Date().getFullYear()
 const RECENT_YEARS = [CURRENT_YEAR - 1, CURRENT_YEAR]
@@ -75,7 +76,7 @@ const communityGetCitationsScore = (aggs: ElasticAggregations): number =>
   communityGetCitationsRecent(aggs) / communityGetPublicationsCount(aggs)
 
 const communityGetDomains = (aggs: ElasticAggregations): Record<string, number> =>
-  aggs?.domains?.buckets.reduce((acc, bucket) => ({ ...acc, [labelClean(bucket.key)]: bucket.doc_count }), {})
+  aggs?.domains?.buckets.reduce((acc, bucket) => ({ ...acc, [labelClean(String(bucket.key))]: bucket.doc_count }), {})
 
 const communityGetOaPercent = (aggs: ElasticAggregations): number => {
   const isOa = aggs?.isOa?.buckets.find((bucket) => bucket.key_as_string === "true")?.doc_count || 0

client/src/api/networks/network/config.ts

@@ -176,8 +176,7 @@ const configGetLinkDescription = (model: string) =>
 
 export function configGetItemUrl(model: string, key: string, label: string): string {
   const targetUrl = CONFIG?.[model]?.url(key, label) ?? ""
-  const baseUrl = window?.location?.href?.split("/networks")[0] ?? ""
-  return baseUrl + targetUrl
+  return window.location.origin + targetUrl
 }
 
 export default function configCreate(model: string): NetworkConfig {

client/src/api/networks/network/ignore.ts

@@ -0,0 +1,76 @@
+export const ignoreIds = {
+  domains: [
+    "Q3526257", // Thèse d'exercice
+    "Q913404", // Thèse d'exercice de médecine
+    "Q669220", // Not available
+    "Q37033", // w3.org
+    "Q466", // www
+    "Q823982", // MathML
+    "Q142", // française
+  ],
+}
+
+export const institutionsAcronyms = {
+  "180089013": "CNRS",
+  "180036048": "INSERM",
+  "775685019": "CEA",
+  "180006025": "IRD",
+  "180070039": "INRAE",
+  "180089047": "INRIA",
+  "775665912": "CNES",
+  "330715368": "IFREMER",
+  "193112562": "ENAC",
+  "195922846R": "BRGM",
+  "775722879": "ONERA",
+  "775688229": "CSTB",
+  "331596270": "CIRAD",
+  "202224326A": "INED",
+  "385290309": "ADEME",
+  "130012024": "ANSES",
+  "130026149": "Univ. PSL",
+  "196901920": "INSA Lyon",
+  "196727671": "INSA Strasbourg",
+  "193500972": "INSA Rennes",
+  "193101524": "INSA Toulouse",
+  "130025752": "INSA HDF",
+  "267500452": "APHP",
+  "261300081": "APHM",
+  "197534712": "CNAM",
+}
+
+const institutionsShort = {
+  "communauté d'universités et établissements": "COMUE",
+  "communauté d'universités et d'établissements": "COMUE",
+  "centre hospitalier régional universitaire": "CHRU",
+  "centre hospitalier et universitaire": "CHU",
+  "centre hospitalier universitaire": "CHU",
+  "centre hospitalier régional": "CHR",
+  "centre hospitalier": "CH",
+  université: "Univ.",
+  universite: "Univ.",
+  university: "Univ.",
+  institute: "Inst.",
+  institut: "Inst.",
+  laboratoire: "Lab.",
+  laboratory: "Lab.",
+  "école normale supérieure de": "ENS",
+  "ecole normale superieure de": "ENS",
+}
+
+function adjustCase(match: string, replacement: string) {
+  if (match === match.toUpperCase()) {
+    return replacement.toUpperCase()
+  } else if (match === match.toLowerCase()) {
+    return replacement.toLowerCase()
+  } else {
+    return replacement
+  }
+}
+
+export function institutionsReplaceLabel(label: string) {
+  Object.entries(institutionsShort).forEach(([key, value]) => {
+    const regex = new RegExp(key, "gi")
+    label = label.replace(regex, (match) => adjustCase(match, value))
+  })
+  return label
+}

client/src/api/networks/network/mistralai.ts

@@ -92,3 +92,28 @@ export async function openAiLabeledClusters(clusters: NetworkCommunities): Promi
 
   return clusters
 }
+
+
+export async function mistralAgentCompletion({ query, agentId }: { query: string; agentId: string }): Promise<unknown> {
+  const chatBody = {
+    messages: [
+      {
+        role: "user",
+        content: query,
+      },
+    ],
+    agent_id: agentId,
+  }
+
+  const response = await fetch(`${MISTRAL_URL}/agents/completions`, {
+    method: "POST",
+    headers: postHeaders,
+    body: JSON.stringify(chatBody),
+  })
+  const completion = await response.json()
+  const answer: string = completion?.choices?.[0]?.message?.content || null
+
+  const json = answer ? JSON.parse(answer) : undefined
+
+  return json
+}

client/src/api/networks/network/network.ts

@@ -4,10 +4,14 @@ import { connectedComponents } from "graphology-components"
 import circular from "graphology-layout/circular"
 import forceAtlas2 from "graphology-layout-forceatlas2"
 import betweenessCentrality from "graphology-metrics/centrality/betweenness"
-import { ElasticBuckets, NetworkFilters, NetworkData, NetworkParameters } from "../../../types/network"
+import { NetworkFilters, NetworkData, NetworkParameters } from "../../../types/network"
 import communitiesCreate from "./communities"
 import { configGetItemUrl } from "./config"
 import { getParameters } from "./parameters"
+import { ElasticAggregation, ElasticBucket } from "../../../types/commons"
+import { ignoreIds, institutionsAcronyms, institutionsReplaceLabel } from "./ignore"
+
+type NetworkBucket = ElasticBucket & { max_year: ElasticAggregation }
 
 const nodeConcatMaxYear = (nodeMaxYear: number, maxYear: number) => (nodeMaxYear ? Math.max(nodeMaxYear, maxYear) : maxYear)
 export const nodeGetId = (id: string) => {
@@ -25,8 +29,7 @@ export default async function networkCreate(
   query: string,
   model: string,
   filters: NetworkFilters,
-  aggregation: ElasticBuckets,
-  computeClusters: boolean,
+  aggregation: Array<NetworkBucket>,
   parameters: NetworkParameters,
   lang: string
 ): Promise<NetworkData> {
@@ -36,13 +39,16 @@ export default async function networkCreate(
   graph.setAttribute("model", model)
   graph.setAttribute("filters", filters)
 
-  const { maxNodes, maxComponents, layout, filterNode, clusters } = getParameters(parameters)
+  const { maxNodes, maxComponents, filterNode, clusters } = getParameters(parameters)
 
   aggregation.forEach((item) => {
     const { key, doc_count: count } = item
     const maxYear = item.max_year?.value
     const nodes = key.split("---")
 
+    // Remove ignored ids
+    if (ignoreIds?.[model]?.includes(nodeGetId(nodes[0])) || ignoreIds?.[model]?.includes(nodeGetId(nodes[1]))) return
+
     // Add nodes and compute weight
     nodes.forEach((id: string) =>
       graph.updateNode(nodeGetId(id), (attr) => ({
@@ -60,6 +66,14 @@ export default async function networkCreate(
     }))
   })
 
+  // Replace institutions labels
+  if (["institutions", "structures"].includes(model)) {
+    graph.updateEachNodeAttributes((node, attr) => ({
+      ...attr,
+      label: institutionsAcronyms?.[node] || institutionsReplaceLabel(attr.label),
+    }))
+  }
+
   // Filter nodes
   if (filterNode) {
     graph = subgraph(graph, [...graph.neighbors(filterNode), filterNode])
@@ -89,19 +103,20 @@ export default async function networkCreate(
   forceAtlas2.assign(graph, { iterations: 100, settings: sensibleSettings })
 
   // Add communities
-  const communities = await communitiesCreate(graph, computeClusters)
+  const communities = await communitiesCreate(graph, clusters)
 
   // Create network
   const network: NetworkData = {
     items: graph.mapNodes((key, attr) => ({
       id: key,
-      ...(layout === "forceatlas" && { x: attr.x, y: attr.y }),
+      x: attr.x,
+      y: attr.y,
       label: attr.label,
-      cluster: attr?.community + 1,
+      cluster: attr.community + 1,
       weights: {
         Weight: attr.weight,
         Degree: graph.degree(key),
-        ...(clusters && { Citations: attr.citationsCount || 0 }),
+        ...(clusters && { Citations: attr?.citationsCount || 0 }),
       },
       scores: { ...(attr?.maxYear && { "Last publication": attr.maxYear }) },
       page: configGetItemUrl(model, key, attr.label),

client/src/api/networks/network/parameters.ts

@@ -19,9 +19,6 @@ export const NETWORK_PARAMETERS: Record<string, PARAMETER> = {
   clusters: {
     default: false,
   },
-  layout: {
-    default: "forceatlas",
-  },
   filterNode: {
     default: "",
   },

client/src/api/networks/search/organization.ts

@@ -3,7 +3,7 @@ import { networkSearch } from "./search"
 
 export async function getStructureNetworkById(id: string, model: string): Promise<Network> {
   const organizationFilter = { terms: { "affiliations.id.keyword": [id] } }
-  const data = networkSearch({ model: model, filters: [organizationFilter] })
+  const data = networkSearch({ model: model, filters: [organizationFilter], lang: "fr" })
 
   return data
 }

client/src/api/networks/search/search.ts

@@ -1,16 +1,10 @@
 import { postHeaders } from "../../../config/api"
-import {
-  Network,
-  NetworkSearchBody,
-  NetworkSearchArgs,
-  ElasticHits,
-  NetworkSearchHitsArgs,
-  ElasticAggregations,
-} from "../../../types/network"
+import { Network, NetworkSearchBody, NetworkSearchArgs, ElasticHits, NetworkSearchHitsArgs } from "../../../types/network"
 import { CONFIG } from "../network/config"
 import networkCreate from "../network/network"
 import configCreate from "../network/config"
 import infoCreate from "../network/info"
+import { ElasticAggregations } from "../../../types/commons"
 
 const CURRENT_YEAR = new Date().getFullYear()
 const DEFAULT_YEARS = Array.from({ length: (2010 - CURRENT_YEAR) / -1 + 1 }, (_, i) => CURRENT_YEAR + i * -1)
@@ -39,7 +33,7 @@ const networkSearchBody = (model: string, query?: string | unknown): NetworkSear
   },
 })
 
-export async function networkSearch({ model, query, options, parameters, filters }: NetworkSearchArgs): Promise<Network> {
+export async function networkSearch({ model, query, lang, parameters, filters }: NetworkSearchArgs): Promise<Network> {
   const body = networkSearchBody(model, query)
 
   if (filters && filters.length > 0) body.query.bool.filter = filters
@@ -64,9 +58,7 @@ export async function networkSearch({ model, query, options, parameters, filters
     return null
   }
 
-  const computeClusters = options?.computeClusters ?? false
-  const lang = options?.lang ?? "fr"
-  const network = await networkCreate(query, model, filters, aggregation, computeClusters, parameters, lang)
+  const network = await networkCreate(query, model, filters, aggregation, parameters, lang)
   const config = configCreate(model)
   const info = infoCreate(query, model)
 

client/src/api/trends/_utils/regression.ts

@@ -0,0 +1,10 @@
+import { SimpleLinearRegression } from "ml-regression-simple-linear"
+
+export const linearRegressionSlope = (values, years) => {
+  const X = years?.map((year) => year - years[0])
+  const y = years?.map((year) => values?.[year] || 0.0)
+  const regression = new SimpleLinearRegression(X, y)
+  const scores = regression.score(X, y)
+
+  return { slope: regression?.slope, intercept: regression?.intercept, r2: scores.r2 }
+}

client/src/api/trends/_utils/variation.ts

@@ -0,0 +1,9 @@
+export default function variation(count, max_year) {
+  const startValue = count?.[max_year - 1] || 0
+  const endValue = count?.[max_year] || 0
+
+  if (!startValue && !endValue) return 0
+  if (!startValue) return Infinity
+
+  return (endValue - startValue) / startValue
+}

client/src/api/trends/config.ts

@@ -0,0 +1,8 @@
+export const CONFIG = {
+  "entity-fishing": {
+    field: "domains.id_name.keyword",
+  },
+  "open-alex": {
+    field: "topics.display_name.keyword",
+  },
+}