From 81e731c10e9d7ca44a8464aff5da7f6c081d8440 Mon Sep 17 00:00:00 2001 From: Hk669 Date: Mon, 1 Jul 2024 17:26:33 +0530 Subject: [PATCH] status: api is working but not chromadb --- chroma/db.py | 5 ++--- src/api.py | 13 ++++++++----- src/search.py | 19 ++++++++++++------- src/user_data.py | 21 +++++++++++++++------ 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/chroma/db.py b/chroma/db.py index 10d8042..7b47c73 100644 --- a/chroma/db.py +++ b/chroma/db.py @@ -1,7 +1,6 @@ import chromadb import random -import os -import sqlite3 + # Recommendations def recommend(user_details, repos): @@ -28,7 +27,7 @@ def recommend(user_details, repos): new_doc = f"{user_proj['project_name']} : {user_proj['description']}" results = collection.query( query_texts = [new_doc], - n_results = 2, + n_results = 3, ) try: # recommending the repos in random diff --git a/src/api.py b/src/api.py index 8dff08e..5422d2a 100644 --- a/src/api.py +++ b/src/api.py @@ -13,13 +13,16 @@ class User(BaseModel): @app.post('/recommendations/') -async def get_recommendations(request : User) -> dict: - username = request.username +async def get_recommendations(user: User) -> dict: + username = user.username try: + print(f'Fetching recommendations for {username}') user_details, language_topics = await get_repos(username) + print(f'--------\n{user_details}') + print(f'--------\n{language_topics}') unique_repos = await main(language_topics) - + print(f'--------\n{unique_repos}') urls = recommend(user_details,unique_repos) return {'recommendations': urls} except Exception as e: @@ -27,9 +30,9 @@ async def get_recommendations(request : User) -> dict: raise HTTPException(status_code=500, detail = 'Error generating recommendatoins') -async def main(): +async def run_server(): uvicorn.run(app, host='0.0.0.0', port=8000) if __name__ == '__main__': - asyncio.run(main()) \ No newline at end of file + asyncio.run(run_server()) \ No newline at end of file diff --git a/src/search.py b/src/search.py index d77b32c..227b099 100644 --- a/src/search.py +++ b/src/search.py @@ -64,11 +64,12 @@ async def main(language_topics): tasks = [] for language in languages: + print(f"Searching for {language} repositories") base_params = { 'q': f'stars:>=2000 forks:>=500 language:{language} pushed:>=2024-01-01', 'sort': 'stars', 'order': 'desc', - 'per_page': 100, + 'per_page': 30, 'page': 1, } @@ -81,11 +82,12 @@ async def main(language_topics): tasks.append(asyncio.create_task(search_repositories(octokit, good_first_issues_params))) for topic in topics: + print(f"Searching for {topic} repositories") base_params = { 'q': f'stars:>=2000 forks:>=500 topic:{topic} pushed:>=2024-01-01', 'sort': 'stars', 'order': 'desc', - 'per_page': 100, + 'per_page': 30, 'page': 1, } @@ -101,19 +103,22 @@ async def main(language_topics): for result in results: unique_repos.update(result) - + print(f"Found {len(unique_repos)} unique repositories") return unique_repos if __name__ == '__main__': - language_topics = { - 'languages': ['python'], - 'topics': ['ai', 'web-development'] - } + language_topics = {'languages': ['Jupyter Notebook', 'Python', 'C++', 'go'], + 'topics': ['openai', 'llm-agent', 'agentic-agi', 'agentic']} + + import time + start = time.time() loop = asyncio.get_event_loop() result = loop.run_until_complete(main(language_topics)) loop.close() + end = time.time() + print(f"Time taken: {end - start:.2f} seconds") for repo_id, repo_info in result.items(): diff --git a/src/user_data.py b/src/user_data.py index 8018ae6..1f9f7c2 100644 --- a/src/user_data.py +++ b/src/user_data.py @@ -10,6 +10,9 @@ GPAT = os.getenv('GPAT') async def get_repos(username: str): + """ + Fetches the repositories of the user + """ user_details = [] language_topics = {} @@ -17,9 +20,9 @@ async def get_repos(username: str): async with ClientSession() as session: octokit = Octokit(GPAT, session) - # To store the unique data of the user - languages_set = set() - topics_set = set() + # To store the unique data of the user + languages_map = {} # store the freq of the languages + topics_map = {} url = f'/users/{username}/repos' repos_data = await octokit.request('GET', url) @@ -36,10 +39,16 @@ async def get_repos(username: str): } user_details.append(user_repo) - languages_set.update(languages_data.keys()) - topics_set.update(repo['topics']) + for lang in languages_data.keys(): + languages_map[lang] = languages_map.get(lang, 0) + 1 - language_topics = {"languages" : list(languages_set), "topics" : list(topics_set)} + for topic in repo['topics']: + topics_map[topic] = topics_map.get(topic, 0) + 1 + + # return the top5 languages + top5_languages = sorted(languages_map, key=languages_map.get, reverse=True)[:5] + top_topics = sorted(topics_map, key=topics_map.get, reverse=True)[:7] + language_topics = {"languages" : list(top5_languages), "topics" : list(top_topics)} except aiohttp.ClientError as e: print(f"Error fetching data: {e}")