add logging

Author: Kaltsoon

.env.template

@@ -6,4 +6,5 @@ POSTGRES_DB=postgres
 REDIS_URL=redis://redis:6379
 OLLAMA_URL=http://host.docker.internal:11434
 OLLAMA_MODEL=llama3
-API_PORT=3000
+API_PORT=3000
+LOGGER_LEVEL=error

README.md

@@ -50,4 +50,8 @@ curl -X POST http://http://localhost:3000/api/repos/Kaltsoon/rag-my-docs \
 
 ### CLI
 
-You can run the CLI application by running `docker compose exec -it app npm run start:cli`. The CLI application will provide instructions on the usage.
+You can run the CLI application by running `docker compose exec -it app npm run start:cli`. The CLI application will provide instructions on the usage.
+
+### Debugging
+
+Setting `LOGGER_LEVEL=debug` in the `.env` file will enable logging and provide information on, for example, cache performance.

package-lock.json

@@ -29,6 +29,7 @@
     "octokit": "^5.0.3",
     "ora": "^8.2.0",
     "pg": "^8.16.3",
+    "pino": "^9.7.0",
     "redis": "^5.6.0",
     "ts-node-dev": "^2.0.0",
     "uuid-by-string": "^4.0.0",

package.json

@@ -8,8 +8,13 @@ import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
 import { Document } from "@langchain/core/documents";
 import getUuidByString from "uuid-by-string";
 
-import { DocumentationFile, RepositoryIdentifier, RepositoryIdentifierWithAuth } from "./types";
+import {
+  DocumentationFile,
+  RepositoryIdentifier,
+  RepositoryIdentifierWithAuth,
+} from "./types";
 import { getRepositoryContentZip } from "./githubApi";
+import { logger } from "./logger";
 
 const textSplitter = RecursiveCharacterTextSplitter.fromLanguage("markdown", {
   chunkSize: 2000,
@@ -32,6 +37,7 @@ export class RepositoryDocumentationLoader {
     const cacheKey = this.#getCacheKey();
 
     if (cache.has(cacheKey)) {
+      logger.info(prefixLoggerMessage(`Cache hit for key ${cacheKey}`));
       return cache.get(cacheKey)!;
     }
 
@@ -44,6 +50,12 @@ export class RepositoryDocumentationLoader {
       documentationFiles
     );
 
+    logger.info(
+      prefixLoggerMessage(
+        `Cache miss for key ${cacheKey}, ${documents.length} documents loaded`
+      )
+    );
+
     cache.set(cacheKey, documents);
 
     return documents;
@@ -56,10 +68,17 @@ export class RepositoryDocumentationLoader {
   }
 }
 
-function getRepositoryFilePath(fullPath: string) {
+function prefixLoggerMessage(message: string) {
+  return `Repository documentation loader: ${message}`;
+}
+
+function getRepositoryFilePath(fullPath: string, tempFolderName: string) {
   const pathParts = fullPath.split(pathSeparator);
+  const tempFolderIndex = pathParts.findIndex(
+    (part) => part === tempFolderName
+  );
 
-  return pathParts.slice(3, pathParts.length).join("/");
+  return pathParts.slice(tempFolderIndex + 2, pathParts.length).join("/");
 }
 
 function getTempRepositoryFolderName(owner: string, repo: string) {
@@ -81,7 +100,7 @@ export async function fetchRepositoryDocumentation(
   const documentationFiles: DocumentationFile[] = [];
 
   for (const filePath of await glob(`${path}/**/*.md`)) {
-    const repositoryFilePath = getRepositoryFilePath(filePath);
+    const repositoryFilePath = getRepositoryFilePath(filePath, tempFolderName);
     const content = await readFile(filePath, "utf-8");
 
     documentationFiles.push({

src/RepositoryDocumentationLoader.ts

@@ -10,6 +10,7 @@ const EnvSchema = z.object({
   OLLAMA_URL: z.string(),
   OLLAMA_MODEL: z.string(),
   API_PORT: z.coerce.number().int(),
+  LOGGER_LEVEL: z.string().default("silent"),
 });
 
 export const env = EnvSchema.parse(process.env);

src/env.ts

@@ -0,0 +1,6 @@
+import pino from "pino";
+import { env } from "./env";
+
+export const logger = pino({
+  level: env.LOGGER_LEVEL,
+});

src/logger.ts

@@ -5,6 +5,7 @@ import { Document } from "@langchain/core/documents";
 import { env } from "./env";
 import { RepositoryIdentifier } from "./types";
 import { setRepositoryDocumentIds, getRepositoryDocumentIds } from "./redis";
+import { logger } from "./logger";
 
 const embeddings = new OllamaEmbeddings({
   model: env.OLLAMA_MODEL,
@@ -32,6 +33,8 @@ const config = {
 
 let cachedVectorStore: PGVectorStore;
 
+logger.info("Hello world!")
+
 export async function addDocuments(documents: Document[]) {
   const vectorStore = await getVectorStore();
   await vectorStore.addDocuments(documents);
@@ -58,20 +61,34 @@ export async function repositorySimilaritySearch(
   });
 }
 
-export async function updateRepositoryVectorStore(repository: RepositoryIdentifier, documents: Document[]) {
+export async function updateRepositoryVectorStore(
+  repository: RepositoryIdentifier,
+  documents: Document[]
+) {
   const nextDocumentIds = documents.map((doc) => doc.id!);
   const previousDocumentIds = await getRepositoryDocumentIds(repository);
 
   if (!documentIdsAreEqual(nextDocumentIds, previousDocumentIds)) {
+    logger.info(prefixLoggerMessage("Cache miss"), {
+      repository,
+      nextDocumentIds,
+      previousDocumentIds,
+    });
+
     if (previousDocumentIds.length > 0) {
+      logger.info(prefixLoggerMessage("Deleting previous documents"), {
+        repository,
+      });
       await deleteDocuments(previousDocumentIds);
     }
 
-    await Promise.all([
+    return Promise.all([
       addDocuments(documents),
       setRepositoryDocumentIds(repository, documents),
     ]);
   }
+
+  logger.info(prefixLoggerMessage("Cache hit"), { repository })
 }
 
 function documentIdsAreEqual(ids: string[], idsOther: string[]) {
@@ -87,3 +104,7 @@ async function getVectorStore() {
 
   return cachedVectorStore;
 }
+
+function prefixLoggerMessage(message: string) {
+  return `Vector store: ${message}`;
+}