From 65fa9dc98b1ce55cdd70d03023da6e2f273f2bda Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 16:40:32 +0530
Subject: [PATCH 01/17] first stab at script

---
 .github/workflows/export_and_update.yml | 99 +++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 .github/workflows/export_and_update.yml

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
new file mode 100644
index 000000000..34f77694f
--- /dev/null
+++ b/.github/workflows/export_and_update.yml
@@ -0,0 +1,99 @@
+name: Consolidate and Upload Documentation
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  consolidate-and-upload:
+    runs-on: ubuntu-latest
+
+    steps:
+      # Step 1: Checkout repository
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Step 2: Set up Python environment
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9" # Use your preferred version
+
+      # Step 3: Install dependencies
+      - name: Install dependencies
+        run: pip install openai
+
+      # Step 4: Consolidate Markdown Files
+      - name: Consolidate Markdown Files
+        run: |
+          python - <<EOF
+          import os
+          import re
+
+          source_dir = "./"  # Current repo directory
+          output_file = "merged_documentation.md"
+          base_url = "https://glific.github.io/docs/docs/"  # Replace with your GitHub Pages URL
+
+          def clean_serial_numbers(name):
+              return re.sub(r'^\d+\.\s*', '', name)
+
+          with open(output_file, "w") as merged_file:
+              merged_file.write("# Consolidated Documentation\n\n")
+
+          for root, _, files in os.walk(source_dir):
+              for file in files:
+                  if file.endswith(".md"):
+                      relative_path = os.path.relpath(os.path.join(root, file), source_dir)
+                      clean_path_parts = [
+                          clean_serial_numbers(part) for part in relative_path.split(os.sep)
+                      ]
+                      relative_path_cleaned = "/".join(clean_path_parts)
+                      relative_path_url = os.path.splitext(relative_path_cleaned)[0].replace(" ", "%20")
+                      page_url = f"{base_url}/{relative_path_url}"
+
+                      page_title = clean_serial_numbers(os.path.splitext(file)[0])
+
+                      with open(output_file, "a") as merged_file:
+                          merged_file.write(f"## [{page_title}]({page_url})\n\n")
+                      
+                      with open(os.path.join(root, file), "r") as md_file:
+                          content = md_file.read()
+                          with open(output_file, "a") as merged_file:
+                              merged_file.write(content + "\n\n---\n\n")
+          EOF
+
+      # Step 5: Upload to Vector Store
+      - name: Upload to Vector Store
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          VECTOR_STORE_ID: ${{ secrets.VECTOR_STORE_ID }}
+        run: |
+          python - <<EOF
+          import openai
+          import os
+
+          openai.api_key = os.getenv("OPENAI_API_KEY")
+          vector_store_id = os.getenv("VECTOR_STORE_ID")
+
+          # Read the consolidated markdown file
+          with open("merged_documentation.md", "r") as file:
+              content = file.read()
+
+          # Upload the consolidated file to the OpenAI vector store
+          response = openai.File.create(
+              file=open("merged_documentation.md", "rb"),
+              purpose="fine-tune"  # or "search", depending on your vector store requirements
+          )
+
+          file_id = response["id"]
+          print("File uploaded successfully with ID:", file_id)
+
+          # Optionally associate the file with a vector store
+          if vector_store_id:
+              print(f"Associating file {file_id} with vector store {vector_store_id}...")
+              # Replace this block with the appropriate call to link file to vector store
+              # For example, use the OpenAI API or custom API for your vector store
+              # Example:
+              # response = some_vector_store_api.add_file_to_vector_store(file_id, vector_store_id)
+              print("File successfully associated.")
+          EOF

From 7034596d0e68e4fc99ea0db2a1035125a592fd33 Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 16:59:04 +0530
Subject: [PATCH 02/17] fixed errors

---
 .github/workflows/export_and_update.yml | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index 34f77694f..946a55a3d 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -72,17 +72,20 @@ jobs:
           import openai
           import os
 
-          openai.api_key = os.getenv("OPENAI_API_KEY")
+          api_key = os.getenv("OPENAI_API_KEY")
           vector_store_id = os.getenv("VECTOR_STORE_ID")
 
+          client = OpenAI(api_key)  # Replace with your actual key
+
           # Read the consolidated markdown file
           with open("merged_documentation.md", "r") as file:
               content = file.read()
 
           # Upload the consolidated file to the OpenAI vector store
-          response = openai.File.create(
-              file=open("merged_documentation.md", "rb"),
-              purpose="fine-tune"  # or "search", depending on your vector store requirements
+
+          response = client.files.create(
+          file=open("/content/merged_documentation.md", "rb"),
+          purpose="assistants"
           )
 
           file_id = response["id"]
@@ -90,10 +93,8 @@ jobs:
 
           # Optionally associate the file with a vector store
           if vector_store_id:
-              print(f"Associating file {file_id} with vector store {vector_store_id}...")
-              # Replace this block with the appropriate call to link file to vector store
-              # For example, use the OpenAI API or custom API for your vector store
-              # Example:
-              # response = some_vector_store_api.add_file_to_vector_store(file_id, vector_store_id)
-              print("File successfully associated.")
+              client.beta.vector_stores.files.create(
+                vector_store_id=vector_store_id,
+                file_id=file.id
+            )
           EOF

From c95b424dd9a28ed494973e764cce809f33a0ef59 Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 17:00:18 +0530
Subject: [PATCH 03/17] fixed imports

---
 .github/workflows/export_and_update.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index 946a55a3d..50b49d962 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -69,7 +69,7 @@ jobs:
           VECTOR_STORE_ID: ${{ secrets.VECTOR_STORE_ID }}
         run: |
           python - <<EOF
-          import openai
+          from openai import OpenAI
           import os
 
           api_key = os.getenv("OPENAI_API_KEY")

From 1df1fdb3f37e2905efa1401b08fea76ce737f75a Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 22:25:30 +0530
Subject: [PATCH 04/17] updated openai functions

---
 .github/workflows/export_and_update.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index 50b49d962..1733d3133 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -69,13 +69,13 @@ jobs:
           VECTOR_STORE_ID: ${{ secrets.VECTOR_STORE_ID }}
         run: |
           python - <<EOF
-          from openai import OpenAI
+          import openai
           import os
 
           api_key = os.getenv("OPENAI_API_KEY")
           vector_store_id = os.getenv("VECTOR_STORE_ID")
 
-          client = OpenAI(api_key)  # Replace with your actual key
+          openai.api_key = api_key
 
           # Read the consolidated markdown file
           with open("merged_documentation.md", "r") as file:
@@ -83,17 +83,19 @@ jobs:
 
           # Upload the consolidated file to the OpenAI vector store
 
-          response = client.files.create(
+          response = openai.File.create(
           file=open("/content/merged_documentation.md", "rb"),
           purpose="assistants"
           )
 
+          print(response)
+
           file_id = response["id"]
           print("File uploaded successfully with ID:", file_id)
 
           # Optionally associate the file with a vector store
           if vector_store_id:
-              client.beta.vector_stores.files.create(
+              openai.vector_stores.files.create(
                 vector_store_id=vector_store_id,
                 file_id=file.id
             )

From 6f9b5850123e6f5649699f39f7dda53bdd0d8ecf Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 22:27:20 +0530
Subject: [PATCH 05/17] updated file path

---
 .github/workflows/export_and_update.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index 1733d3133..9d7731fad 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -84,7 +84,7 @@ jobs:
           # Upload the consolidated file to the OpenAI vector store
 
           response = openai.File.create(
-          file=open("/content/merged_documentation.md", "rb"),
+          file=open("merged_documentation.md", "rb"),
           purpose="assistants"
           )
 

From 28c2cda0e1f08128d30614d3af3aa46bb40c082d Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 22:34:41 +0530
Subject: [PATCH 06/17] updated file path

---
 .github/workflows/export_and_update.yml | 22 +++++++---------------
 1 file changed, 7 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index 9d7731fad..7e9fd496a 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -69,13 +69,14 @@ jobs:
           VECTOR_STORE_ID: ${{ secrets.VECTOR_STORE_ID }}
         run: |
           python - <<EOF
-          import openai
+          from openai import OpenAI
           import os
 
-          api_key = os.getenv("OPENAI_API_KEY")
           vector_store_id = os.getenv("VECTOR_STORE_ID")
 
-          openai.api_key = api_key
+          client = OpenAI(
+            api_key=os.getenv("OPENAI_API_KEY")
+          )
 
           # Read the consolidated markdown file
           with open("merged_documentation.md", "r") as file:
@@ -83,20 +84,11 @@ jobs:
 
           # Upload the consolidated file to the OpenAI vector store
 
-          response = openai.File.create(
-          file=open("merged_documentation.md", "rb"),
-          purpose="assistants"
+          response = openAI.files.create(
+            file=open("merged_documentation.md", "rb"),
+            purpose="assistants"
           )
 
           print(response)
 
-          file_id = response["id"]
-          print("File uploaded successfully with ID:", file_id)
-
-          # Optionally associate the file with a vector store
-          if vector_store_id:
-              openai.vector_stores.files.create(
-                vector_store_id=vector_store_id,
-                file_id=file.id
-            )
           EOF

From 38a04d3ed63421394eb6bc4da403fa1de2e9e728 Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 22:36:00 +0530
Subject: [PATCH 07/17] updated openai imports

---
 .github/workflows/export_and_update.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index 7e9fd496a..cb3086a42 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -84,7 +84,7 @@ jobs:
 
           # Upload the consolidated file to the OpenAI vector store
 
-          response = openAI.files.create(
+          response = client.files.create(
             file=open("merged_documentation.md", "rb"),
             purpose="assistants"
           )

From 3e23a3a62425865f9035d386e7c4549c95c722ea Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 22:39:16 +0530
Subject: [PATCH 08/17] attaching file to vector store

---
 .github/workflows/export_and_update.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index cb3086a42..bd8413eed 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -91,4 +91,14 @@ jobs:
 
           print(response)
 
+          file_id = response["id"]
+          print("File uploaded successfully with ID:", file_id)
+
+          # Optionally associate the file with a vector store
+          if vector_store_id:
+              client.vector_stores.files.create(
+                vector_store_id=vector_store_id,
+                file_id=file.id
+            )
+
           EOF

From f35e174014b5069af1367d41789ca962e3579e0f Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 22:39:59 +0530
Subject: [PATCH 09/17] attaching file to vector store

---
 .github/workflows/export_and_update.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index bd8413eed..b224d3203 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -91,7 +91,7 @@ jobs:
 
           print(response)
 
-          file_id = response["id"]
+          file_id = response.id
           print("File uploaded successfully with ID:", file_id)
 
           # Optionally associate the file with a vector store

From c5a57def465bf31543882315630194bf9497e185 Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 22:43:57 +0530
Subject: [PATCH 10/17] added beta

---
 .github/workflows/export_and_update.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index b224d3203..fae7209d1 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -96,7 +96,7 @@ jobs:
 
           # Optionally associate the file with a vector store
           if vector_store_id:
-              client.vector_stores.files.create(
+              client.beta.vector_stores.files.create(
                 vector_store_id=vector_store_id,
                 file_id=file.id
             )

From 5923fd7ce05093888b5121ddaf4d472f24d82e42 Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 22:46:37 +0530
Subject: [PATCH 11/17] added beta

---
 .github/workflows/export_and_update.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index fae7209d1..712e5d0e8 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -98,7 +98,7 @@ jobs:
           if vector_store_id:
               client.beta.vector_stores.files.create(
                 vector_store_id=vector_store_id,
-                file_id=file.id
+                file_id=file_id
             )
 
           EOF

From 9da4325e021aab719af10c9c86af4c2e11af7dfc Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 22:53:47 +0530
Subject: [PATCH 12/17] functionality to remove older files

---
 .github/workflows/export_and_update.yml | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index 712e5d0e8..57cde319a 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -94,11 +94,22 @@ jobs:
           file_id = response.id
           print("File uploaded successfully with ID:", file_id)
 
-          # Optionally associate the file with a vector store
-          if vector_store_id:
-              client.beta.vector_stores.files.create(
-                vector_store_id=vector_store_id,
-                file_id=file_id
+          if file_id:
+            vector_store_files = client.beta.vector_stores.files.list(
+              vector_store_id = vector_store_id,
             )
 
+            old_file_id = vector_store_files.data[0].id
+            
+            deleted_vector_store_file = client.beta.vector_stores.files.delete(
+              vector_store_id = vector_store_id,
+              file_id = file_id)
+            
+            print("Old file removed:", deleted_vector_store_file)
+
+            if vector_store_id:
+                client.beta.vector_stores.files.create(
+                  vector_store_id=vector_store_id,
+                  file_id=file_id)
+
           EOF

From 5dffc80868554e0cb9249463cac5ef868b5ed3de Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 23:00:52 +0530
Subject: [PATCH 13/17] updated file id

---
 .github/workflows/export_and_update.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index 57cde319a..27f1c21e5 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -96,20 +96,19 @@ jobs:
 
           if file_id:
             vector_store_files = client.beta.vector_stores.files.list(
-              vector_store_id = vector_store_id,
-            )
+              vector_store_id = vector_store_id )
 
             old_file_id = vector_store_files.data[0].id
-            
+
             deleted_vector_store_file = client.beta.vector_stores.files.delete(
               vector_store_id = vector_store_id,
-              file_id = file_id)
+              file_id = old_file_id )
             
             print("Old file removed:", deleted_vector_store_file)
 
             if vector_store_id:
                 client.beta.vector_stores.files.create(
-                  vector_store_id=vector_store_id,
-                  file_id=file_id)
+                  vector_store_id = vector_store_id,
+                  file_id=file_id )
 
           EOF

From 9dd6494db55675580777fdef776db70584179c8b Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 23:12:11 +0530
Subject: [PATCH 14/17] cross checking file

---
 .github/workflows/export_and_update.yml | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index 27f1c21e5..38876a9bf 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -1,11 +1,11 @@
-name: Consolidate and Upload Documentation
+name: Consolidate and Update Documentation
 
 on:
   pull_request:
     branches: [main]
 
 jobs:
-  consolidate-and-upload:
+  consolidate-and-update:
     runs-on: ubuntu-latest
 
     steps:
@@ -62,6 +62,13 @@ jobs:
                               merged_file.write(content + "\n\n---\n\n")
           EOF
 
+      # Step 5: Upload consolidated file as an artifact
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: consolidated-documentation
+          path: merged_documentation.md
+
       # Step 5: Upload to Vector Store
       - name: Upload to Vector Store
         env:
@@ -103,7 +110,9 @@ jobs:
             deleted_vector_store_file = client.beta.vector_stores.files.delete(
               vector_store_id = vector_store_id,
               file_id = old_file_id )
-            
+
+            client.files.delete(old_file_id)
+
             print("Old file removed:", deleted_vector_store_file)
 
             if vector_store_id:

From 8d0b39e8191ea514b743b6d0bf4a26ab211f25ad Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 23:16:06 +0530
Subject: [PATCH 15/17] cross checking file

---
 .github/workflows/export_and_update.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index 38876a9bf..3b8bf9617 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -32,7 +32,7 @@ jobs:
 
           source_dir = "./"  # Current repo directory
           output_file = "merged_documentation.md"
-          base_url = "https://glific.github.io/docs/docs/"  # Replace with your GitHub Pages URL
+          base_url = "https://glific.github.io/docs"  # Replace with your GitHub Pages URL
 
           def clean_serial_numbers(name):
               return re.sub(r'^\d+\.\s*', '', name)

From 9dac885ffe3c3914f5ae5c7325b449835777cc35 Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 23:19:25 +0530
Subject: [PATCH 16/17] cross checking file

---
 .github/workflows/export_and_update.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index 3b8bf9617..dfab23e44 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -62,13 +62,6 @@ jobs:
                               merged_file.write(content + "\n\n---\n\n")
           EOF
 
-      # Step 5: Upload consolidated file as an artifact
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: consolidated-documentation
-          path: merged_documentation.md
-
       # Step 5: Upload to Vector Store
       - name: Upload to Vector Store
         env:

From 1ba6cca3812f1294ffe54665cd1c59ffe8429438 Mon Sep 17 00:00:00 2001
From: Akansha Sakhre <asakhre2002@gmail.com>
Date: Fri, 22 Nov 2024 23:20:57 +0530
Subject: [PATCH 17/17] changed from pr to merge

---
 .github/workflows/export_and_update.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/export_and_update.yml b/.github/workflows/export_and_update.yml
index dfab23e44..06e59f8b5 100644
--- a/.github/workflows/export_and_update.yml
+++ b/.github/workflows/export_and_update.yml
@@ -1,7 +1,7 @@
 name: Consolidate and Update Documentation
 
 on:
-  pull_request:
+  push:
     branches: [main]
 
 jobs: