Extract user metrics from cache (#97)

* Extract user metrics from cache

* Fix prisma files

* Added repo name on metrics

* Fix comments

* Change migration name

.dockerignore

@@ -0,0 +1 @@
+**/node_modules

.env.example

@@ -8,5 +8,6 @@ NEXTAUTH_SECRET=RAMDON_GENERATED_BASE64
 
 CRED_MANAGER_ROUTE="http://localhost:8080"
 SOURCECRED_INSTANCE_PATH="/instance"
+SOURCECRED_CACHE_DB_PATH="/instance/cache/sourcecred/github"
 SENDGRID_API_KEY=REPLACE_WITH_REAL
 

Dockerfile

@@ -6,7 +6,7 @@ RUN apk add git
 COPY cred-manager/package.json /app/package.json
 COPY cred-manager/yarn.lock /app/yarn.lock
 
-RUN yarn install --ignore-scripts --frozen-lockfile
+RUN yarn install --frozen-lockfile
 
 COPY cred-manager/ /app/
 

cred-manager/.env.docker

@@ -8,3 +8,4 @@
 # DATABASE_URL="postgresql://postgres:postgres@localhost:54322/postgres?schema=public"
 DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:54322/postgres?schema=public"
 SOURCECRED_INSTANCE_PATH="/instance"
+SOURCECRED_CACHE_DB_PATH="/instance/cache/sourcecred/github"

cred-manager/package.json

@@ -31,7 +31,8 @@
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
     "rxjs": "^7.2.0",
-    "sourcecred": "^0.11.2"
+    "sourcecred": "^0.11.2",
+    "better-sqlite3": "^7.0.0"
   },
   "devDependencies": {
     "@nestjs/cli": "^8.0.0",

cred-manager/prisma-lite/README.md

@@ -0,0 +1,3 @@
+## This is the prisma schema for sourcecred cache db
+
+Prisma does not have good support for sql lite, so this is put on hold for now

cred-manager/prisma-lite/schema.prisma

@@ -0,0 +1,139 @@
+generator client {
+  provider = "prisma-client-js"
+  output   = "../node_modules/@prisma-lite/client"
+}
+
+datasource db {
+  provider = "sqlite"
+  url      = "file:" + env("SOURCECRED_CACHE_DB_PATH")
+}
+
+model ConnectionEntry {
+  rowid         Int     @id @default(autoincrement())
+  connection_id Int
+  idx           Int
+  child_id      String?
+
+  // Define foreign key relationships
+  connection Connection @relation(fields: [connection_id], references: [rowid])
+  object     Object?    @relation(fields: [child_id], references: [id])
+
+  // Ensure uniqueness of (connection_id, idx) pair
+  @@unique([connection_id, idx])
+
+  @@map("connection_entries")
+}
+
+// Define the Connections model
+model Connection {
+  rowid         Int      @id @default(autoincrement())
+  object_id     String
+  fieldname     String
+  last_update   Int?
+  total_count   Int?
+  has_next_page Boolean?
+  end_cursor    String?
+
+  // Define foreign key relationships
+  object Object  @relation(fields: [object_id], references: [id])
+  update Update? @relation(fields: [last_update], references: [rowid])
+
+  // Define the opposite relation field
+  connection_entries ConnectionEntry[]
+
+  // Ensure uniqueness of (object_id, fieldname) pair
+  @@unique([object_id, fieldname])
+
+  @@map("connections")
+}
+
+// Define the Links model
+model Link {
+  rowid     Int     @id @default(autoincrement())
+  parent_id String
+  fieldname String
+  child_id  String?
+
+  // Define foreign key relationships
+  parentObject Object  @relation("ParentObject", fields: [parent_id], references: [id])
+  childObject  Object? @relation("ChildObject", fields: [child_id], references: [id])
+
+  // Ensure uniqueness of (parent_id, fieldname) pair
+  @@unique([parent_id, fieldname])
+
+  @@map("links")
+}
+
+// Define the Meta model
+model Meta {
+  zero   Int    @id
+  config String
+
+  @@map("meta")
+}
+
+// Define the NetworkLog model
+model NetworkLog {
+  rowid                      Int     @id @default(autoincrement())
+  query                      String?
+  query_parameters           String?
+  request_time_epoch_millis  BigInt?
+  response                   String?
+  response_time_epoch_millis BigInt?
+  update_id                  Int?
+
+  // Define foreign key relationships
+  update Update? @relation(fields: [update_id], references: [rowid])
+
+  @@map("network_log")
+}
+
+// Define the Object model
+model Object {
+  id          String  @id
+  typename    String?
+  last_update Int?
+
+  // Define foreign key relationships
+  lastUpdate Update? @relation(fields: [last_update], references: [rowid])
+
+  // Define the opposite relation field
+  connection_entries ConnectionEntry[]
+  parentLinks        Link[]            @relation("ParentObject")
+  childLinks         Link[]            @relation("ChildObject")
+  connection         Connection[]
+  updates            Update[]          @relation("lastUpdate")
+  Primitive          Primitive[]
+
+  @@map("objects")
+}
+
+// Define the Primitives model
+model Primitive {
+  rowid     Int     @id @default(autoincrement())
+  object_id String
+  fieldname String
+  value     String?
+
+  // Define foreign key relationships
+  object Object @relation(fields: [object_id], references: [id])
+
+  // Ensure uniqueness of (object_id, fieldname) pair
+  @@unique([object_id, fieldname])
+
+  @@map("primitives")
+}
+
+// Define the Updates model
+model Update {
+  rowid             Int @id @default(autoincrement())
+  time_epoch_millis Int
+
+  // Define foreign key relationships
+  connection Connection[]
+  networkLog NetworkLog[]
+  objects    Object[]     @relation("lastUpdate")
+  Object     Object[]
+
+  @@map("updates")
+}

cred-manager/prisma/schema.prisma

@@ -41,12 +41,13 @@ model Team {
 }
 
 model ContributionCalculation {
-  id             String      @id @default(uuid())
+  id             String           @id @default(uuid())
   team_id        String
   score_interval Json?
-  Team           Team        @relation(fields: [team_id], references: [id], onDelete: Cascade)
+  Team           Team             @relation(fields: [team_id], references: [id], onDelete: Cascade)
   UserScore      UserScore[]
-  created_at     DateTime    @default(now()) @db.Timestamp(6)
+  created_at     DateTime         @default(now()) @db.Timestamp(6)
+  UserTeamMetric UserTeamMetric[]
 }
 
 model ContributionRequest {
@@ -105,3 +106,14 @@ model TeamWeightConfig {
   team_id String
   team    Team   @relation(fields: [team_id], references: [id], onDelete: Cascade)
 }
+
+model UserTeamMetric {
+  id                          String                  @id @default(uuid())
+  username                    String                  @db.VarChar(255)
+  contribution_calculation_id String
+  repo_name                   String                  @db.VarChar(1000)
+  metric_name                 String                  @db.VarChar(1000)
+  metric_count                String                  @db.VarChar(1000)
+  created_at                  DateTime                @default(now()) @db.Timestamp(6)
+  contribution_calculation    ContributionCalculation @relation(fields: [contribution_calculation_id], references: [id], onDelete: Cascade)
+}

cred-manager/src/app.controller.ts

@@ -1,7 +1,6 @@
 import { Controller, Get } from '@nestjs/common';
 import { AppService } from './app.service';
 import { PrismaService } from './prisma/prisma.service';
-import fetch from 'node-fetch';
 
 @Controller()
 export class AppController {

cred-manager/src/sourcecred/service/cache.service.ts

@@ -0,0 +1,73 @@
+import { Injectable, Logger } from '@nestjs/common';
+
+const Database = require('better-sqlite3');
+import { UsersContibutionMetricsDto } from '../types/userContibutionMetric.dto';
+import { extname, resolve } from 'path';
+import { readdirSync } from 'fs';
+
+@Injectable()
+export class CacheService {
+  private readonly logger = new Logger(CacheService.name);
+
+  public getSqlLiteDatabase(dbPath: string): any[] {
+    const dbFiles = this.findFilesWithExtension(dbPath, '.db');
+    return dbFiles.map((db) => {
+      return new Database(db);
+    });
+  }
+
+  public async getUsersContributiosnMetrics(
+    database: any,
+  ): Promise<UsersContibutionMetricsDto[]> {
+    const sqlQuery = `
+      select 
+          userDetails.value as username, 
+          artifactsObject.typename as metric, 
+          count(artifactsObject.typename) as count 
+        from objects as userObject  
+          join primitives as userDetails on userDetails.object_id = userObject.id and userDetails.fieldname = ?
+          join links as links on userObject.id = links.child_id and userObject.typename = ?
+          join objects as artifactsObject on  artifactsObject.id =  links.parent_id
+      group by userObject.id,  artifactsObject.typename;
+    `;
+    const rows = database.prepare(sqlQuery).bind('login', 'User').all();
+    return rows.map((row) => ({
+      username: JSON.parse(row.username),
+      metric: row.metric,
+      count: row.count,
+    }));
+  }
+
+  public async getRepositoryName(database: any): Promise<string> {
+    const sqlQuery = `
+    select primitives.value from objects 
+      join primitives on primitives.object_id = objects.id 
+    where objects.typename = ?
+      and fieldname = ?
+    `;
+    const row = database.prepare(sqlQuery).bind('Repository', 'url').all();
+    const parsedUrl = new URL(JSON.parse(row[0].value));
+    const pathname = parsedUrl.pathname.replace('/', '');
+    return pathname;
+  }
+
+  findFilesWithExtension(directory: string, extension: string): string[] {
+    const files: string[] = readdirSync(directory);
+    const filteredFiles: string[] = files.filter(
+      (file) => extname(file) === extension,
+    );
+    return filteredFiles.map((file) => resolve(directory, file));
+  }
+
+  executeSql(sql: string, database: any): Promise<any[]> {
+    return new Promise((resolve, reject) => {
+      database.all(sql, (error: Error | null, rows: any[]) => {
+        if (error) {
+          reject(error);
+          return;
+        }
+        resolve(rows);
+      });
+    });
+  }
+}