annotations.qmd

---
title: "Systematic literature review"
bibliography: references.bib
title-block-banner: true
subtitle: "Annotations"
author:
  - name: Olivier Caron
    email: olivier.caron@dauphine.psl.eu
    affiliations: 
      name: "Paris Dauphine - PSL"
      city: Paris
      state: France
  - name: Christophe Benavent
    email: christophe.benavent@dauphine.psl.eu
    affiliations: 
      name: "Paris Dauphine - PSL"
      city: Paris
      state: France
date : "last-modified"
toc: true
number-sections: true
number-depth: 5
format:
  html:
    theme:
      light: yeti
      dark: darkly
    code-fold: true
    code-summary: "Display code"
    code-tools: true #enables to display/hide all blocks of code
    code-copy: true #enables to copy code
    grid:
      body-width: 1000px
      margin-width: 100px
    toc: true
    toc-location: left
execute:
  echo: true
  warning: false
  message: false
editor: visual
fig-align: "center"
highlight-style: ayu
css: styles.css
reference-location: margin
---

## Libraries and loading data

```{r}
#| label: load-packages
#| message: false

library(tidyverse)
library(word2vec)
library(quanteda)
library(text)
library(udpipe)
library(plotly)
library(text2vec)
library(parallel)
library(reactable)
num_cores <- detectCores(logical = TRUE) - 1
cat('Number of logical cores to use when training Word2vec model: ', num_cores, '\n')
```

## Loading data

```{r}
#| label: load-data


list_articles <- read.csv2("nlp_full_data_final_18-08-2023.csv", encoding = "UTF-8") %>%
  rename("entry_number" = 1)
list_references <- read.csv2("nlp_references_final_18-08-2023.csv", encoding = "UTF-8") %>%
  rename("citing_art" = 1)
colnames(list_articles) <- gsub("\\.+", "_", colnames(list_articles)) # <1>
colnames(list_articles) <- gsub("^[[:punct:]]+|[[:punct:]]+$", "", colnames(list_articles)) # <2>
colnames(list_references) <- gsub("\\.+", "_", colnames(list_references))
colnames(list_references) <- gsub("^[[:punct:]]+|[[:punct:]]+$", "", colnames(list_references))


data_embeddings <- list_articles %>%
  distinct(entry_number, .keep_all = TRUE) %>%
  filter(marketing == 1) %>%
  mutate("year" = substr(prism_coverDate, 7, 10)) %>%
  mutate(keywords = str_replace_all(authkeywords, "\\|", "")) %>%
  mutate(keywords = str_squish(keywords)) %>%
  mutate("combined_text" = paste0(dc_title,". ", dc_description, ". ", keywords))

#write.csv(data_embeddings,"data_for_embeddings.csv")
#data_embeddings <- read.csv("data_for_embeddings.csv")
#embeddings <- read.csv("embeddings_bge.csv")
```

## A glimpse of data

```{r}
#| label: glimpse-data

data_embeddings %>%
  head(5) %>%
  select(entry_number, dc_creator, combined_text, year)
```

## A first Word2Vec embeddings analysis (skip-gram)

```{r}
#| label: word2vec-skipgram
 
set.seed(42)
model <- word2vec(x = tolower(data_embeddings$combined_text),
                  type = "skip-gram",
                  dim = 500,
                  iter = 250,
                  min_count = 3,
                  hs = 1,
                  verbose=10,
                  stopwords = stopwords("english"),
                  window = "7",
                  threads = num_cores)
embedding <- as.matrix(model)


```

### Let's try to find softwares related to text analysis using additions of word embeddings

```{r}
#| label: plotly-embedding-skipgram
#| panel: center

#addition of the wordd embeddings "text", "analysis" and "software"
embedding_text <- predict(model, c("text"), type = "embedding")
embedding_analysis <- predict(model, c("analysis"), type = "embedding")
embedding_software <- predict(model, c("software"), type = "embedding")
embedding_text_software <- embedding_text + embedding_analysis + embedding_software

lookslike <- predict(model, embedding_text_software, type = "nearest", top_n = 200) %>% as.data.frame() %>% mutate(text.similarity = round(text.similarity, 2)) %>% select(text.term, text.similarity)
reactable(lookslike, searchable = TRUE, defaultColDef = colDef(
  maxWidth = 200,
  sortable = TRUE,
  resizable = TRUE
))

fig <- lookslike %>% top_n(20, text.similarity) %>%
  ggplot(aes(x = text.similarity, y = reorder(text.term, text.similarity))) +
  geom_point(color = "royalblue") +
  theme_minimal() +
  labs(x = "similarity score",
       y = "") +
  ggtitle("Top 20 similarity scores with 'text'+'analysis'+'software' embeddings" ) +
  theme(plot.title = element_text(hjust = 0.5))

ggplotly(fig)
```

## A Word2Vec embeddings analysis (cbow)

```{r}
#| label: word2vec-cbow

set.seed(42)
modelcbow <- word2vec(x = tolower(data_embeddings$combined_text),
                  type = "cbow",
                  dim = 500,
                  iter = 250,
                  min_count = 3,
                  hs = 1,
                  verbose=10,
                  stopwords = stopwords("english"),
                  window = "7",
                  threads = num_cores)
embedding1 <- as.matrix(modelcbow)
#embedding <- predict(modelcbow, c("mining"), type = "embedding")
#embedding
```

```{r}
#| label: plotly-embedding-cbow
#| panel: center

#addition of the wordd embeddings "text", "analysis" and "software"
embedding_text <- predict(modelcbow, c("text"), type = "embedding")
embedding_analysis <- predict(modelcbow, c("analysis"), type = "embedding")
embedding_software <- predict(modelcbow, c("software"), type = "embedding")
embedding_text_software <- embedding_text + embedding_analysis + embedding_software

lookslike <- predict(modelcbow, embedding_text_software, type = "nearest", top_n = 200) %>% as.data.frame() %>% mutate(text.similarity = round(text.similarity, 2)) %>% select(text.term, text.similarity)
reactable(lookslike, searchable = TRUE, defaultColDef = colDef(
  maxWidth = 200,
  sortable = TRUE,
  resizable = TRUE
))

fig <- lookslike %>% top_n(20, text.similarity) %>%
  ggplot(aes(x = text.similarity, y = reorder(text.term, text.similarity))) +
  geom_point(color = "royalblue") +
  theme_minimal() +
  labs(x = "similarity score",
       y = "") +
  ggtitle("Top 20 similarity scores with 'text'+'analysis'+'software' embeddings" ) +
  theme(plot.title = element_text(hjust = 0.5))

ggplotly(fig)

```

It seems that skip-gram performs better here. Skip-gram is better at capturing infrequent words, while CBOW is faster and has better representations for more frequent words. Also, skip-gram trains the context word from the center word, while CBOW trains the center word from the context word.

More simply:

-   CBOW model tries to predict the missing word given a set of context words;

-   Skip-gram model tries to predict the other context missing words given a word.

## Part of speech tagging with UDPipe (@straka-strakova-2017-tokenizing)

```{r}
#| label: udpipe
udpipe_download_model(language = "english")
udmodel_english <- udpipe_load_model(file = "english-ewt-ud-2.5-191206.udpipe")
t1=Sys.time()
UD <- udpipe_annotate(udmodel_english, x=data_embeddings$combined_text, trace =40, parallel.cores = 6)
Sys.time()-t1
annotated_text <- UD %>% as.data.frame()
#write.csv(annotated_text,"annotated_udpipe.csv")
```

## Top 20 nouns with UDPipe

```{r}
#| label: plotly-udpipe
lemma <- annotated_text %>%
  filter(upos == "NOUN") %>%
  group_by(lemma) %>%
  summarize(n = n()) %>%
  top_n(20)

fig <- ggplot(lemma, aes(x = n, y = reorder(lemma, n))) +
  geom_point(color = "royalblue") +
  theme_minimal() +
  labs(x = "Frequency",
       y = "Lemma") +
  ggtitle("Top 20 Most Frequent Nouns") +
  theme(plot.title = element_text(hjust = 0.5))

ggplotly(fig)
```

## Part of speech tagging with Trankit (@nguyen2021trankit)

```{python}
#| label: trankit
from trankit import Pipeline
import pandas as pd
from pandas import json_normalize

df = pd.read_csv("data_for_embeddings.csv")

# initialize a pipeline for English
p = Pipeline('english')

#test = p.posdep(df.loc[:5]['combined_text'][1])
#testdf = pd.DataFrame(pd.json_normalize(test['sentences'], 'tokens'))
  
#pos = p.posdep(all)

results = pd.DataFrame()  
#part of speech tagging
for text in df['combined_text']:
    pos = p.posdep(text)
    pos_df = pd.json_normalize(pos['sentences'], 'tokens')
    results = pd.concat([results,pos_df])

#lemmatization
results_lemma = pd.DataFrame()  
for text in df['combined_text']:
    lemma = p.lemmatize(text)
    lemma_df = pd.json_normalize(lemma['sentences'], 'tokens')
    results_lemma = pd.concat([results_lemma,lemma_df])
    
#join both data frames
results_complete = pd.concat([results, results_lemma['text'].rename("lemma")], axis=1)
results_complete["lemma"] = results_complete["text"]
#results_lemma.to_csv("lemmas_trankit.csv")
#results_complete.to_csv("annotated_trankit.csv")
```

## Top 20 nouns with Trankit

```{python}
#| label: plotly-trankit
import pandas as pd
import plotly.express as px

# Load data from the "results" DataFrame
# Ensure that "results" contains the same columns as the original CSV file
# (e.g., "upos" and "lemma")
results_complete = pd.read_csv("annotated_trankit.csv")

# Filter rows where 'upos' is equal to "NOUN"
noun_data = results_complete[results_complete['upos'] == 'NOUN']

# Group by 'lemma' and count the number of occurrences
top_nouns = noun_data['lemma'].value_counts().reset_index()
top_nouns.columns = ['lemma', 'n']
top_nouns = top_nouns.head(20)

# Create a chart using Plotly Express
if "fig" not in globals():
    fig = px.scatter(top_nouns, x='n', y='lemma', color='lemma',
                     labels={'n': 'Frequency', 'lemma': 'Lemma'},
                     title='Top 20 Most Frequent Nouns')
    
    # Customize the chart's style
    fig.update_traces(marker=dict(size=12, opacity=0.6),
                      selector=dict(mode='markers'))
    
    fig.update_layout(title_x=0.5, title_font=dict(size=20))
    fig.update_layout(template="plotly_white")

# Display the chart
#fig.show()
#fig.write_html("top_20_nouns_trankit_python.html")



```

## Part of speech tagging with Stanza (@qi2020stanza)

```{python}
#| label: stanza
import stanza
import pandas as pd
from tqdm import tqdm

# Initialize the Stanza model
nlp = stanza.Pipeline(lang='en', processors='tokenize, mwt, pos, lemma, ner', use_gpu=True, tokenize_pretokenized=False, tokenize_no_ssplit=True)

# Load the DataFrame from the CSV file
df = pd.read_csv("data_for_embeddings.csv")

# Create an empty DataFrame to store annotated data
annotated_df = pd.DataFrame()

for text in tqdm(df['combined_text'], desc="Processing Texts"):
    doc = nlp(text)
    dicts = doc.to_dict()
    
    # Convert the dictionary into a temporary DataFrame
    temp_df = pd.DataFrame(dicts[0])
    
    # Append data from the temporary DataFrame to annotated_df while ignoring the index
    annotated_df = pd.concat([annotated_df, temp_df], ignore_index=True)

# Uncomment the line below to save the annotated data to a CSV file
# annotated_df.to_csv("annotated_stanza.csv")

```

## Top 20 nouns with Stanza

```{r}
#| label: plotly-stanza
annotation_stanza <- read.csv("annotated_stanza.csv")
lemma <- annotation_stanza %>%
  filter(upos == "NOUN") %>%
  group_by(lemma) %>%
  summarize(n = n()) %>%
  top_n(20)

fig <- ggplot(lemma, aes(x = n, y = reorder(lemma, n))) +
  geom_point(color = "royalblue") +
  theme_minimal() +
  labs(x = "Frequency",
       y = "Lemma") +
  ggtitle("Top 20 Most Frequent Nouns") +
  theme(plot.title = element_text(hjust = 0.5))

ggplotly(fig)
```