feat(scanner): integrate buit-in stats in the response of depWalker #612

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

fraxken merged 4 commits into master from add-built-in-stats

Jan 21, 2026

.changeset/wet-bears-notice.md

-Original file line number
+Diff line change
@@ -0,0 +1,5 @@
+    ---
+    "@nodesecure/scanner": minor
+    ---
+    feat(scanner): integrate built-in stats in the response of depWalker

workspaces/scanner/src/class/DateProvider.class.ts

-Original file line number
+Diff line change
@@ -0,0 +1,17 @@
+    export interface DateProvider {
+      now(): number;
+      oneYearAgo(): Date;
+    }
+    export class SystemDateProvider implements DateProvider {
+      now(): number {
+        return Date.now();
+      }
+      oneYearAgo(): Date {
+        const date = new Date();
+        date.setFullYear(date.getFullYear() - 1);
+        return date;
+      }
+    }

workspaces/scanner/src/class/StatsCollector.class.ts

-Original file line number
+Diff line change
@@ -0,0 +1,50 @@
+    // Import Internal Dependencies
+    import { SystemDateProvider, type DateProvider } from "./DateProvider.class.ts";
+    import type { ApiStats, Stats } from "../types.ts";
+    export class StatsCollector {
+      #apiCalls: ApiStats[] = [];
+      #dateProvider: DateProvider;
+      #startedAt: number;
+      constructor(dateProvider: DateProvider = new SystemDateProvider()) {
+        this.#dateProvider = dateProvider;
+        this.#startedAt = this.#dateProvider.now();
+      }
+      track<T extends () => any>(name: string, fn: T): ReturnType<T> {
+        const startedAt = this.#dateProvider.now();
+        try {
+          const result = fn();
+          if (result instanceof Promise) {
+            return result.finally(() => this.#addApiStat(name, startedAt)
+            ) as ReturnType<T>;
+          }
+          this.#addApiStat(name, startedAt);
+          return result;
+        }
+        catch (err) {
+          this.#addApiStat(name, startedAt);
+          throw err;
+        }
+      }
+      #addApiStat(name: string, startedAt: number) {
+        this.#apiCalls.push({
+          name,
+          startedAt,
+          executionTime: this.#dateProvider.now() - startedAt
+        });
+      }
+      getStats(): Stats {
+        return {
+          startedAt: this.#startedAt,
+          executionTime: this.#dateProvider.now() - this.#startedAt,
+          apiCalls: this.#apiCalls,
+          apiCallsCount: this.#apiCalls.length
+        };
+      }
+    }

workspaces/scanner/src/depWalker.ts

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -3,6 +3,8 @@ import path from "node:path";
  
    import { readFileSync } from "node:fs";

    // Import Third-party Dependencies

    import pacote from "pacote";

    import * as npmRegistrySDK from "@nodesecure/npm-registry-sdk";

    import { Mutex, MutexRelease } from "@openally/mutex";

    import {

      extractAndResolve,

    @@ -26,7 +28,8 @@ import {
  
      getManifestLinks,

      NPM_TOKEN

    } from "./utils/index.ts";

    import { NpmRegistryProvider } from "./registry/NpmRegistryProvider.ts";

    import { NpmRegistryProvider, type NpmApiClient } from "./registry/NpmRegistryProvider.ts";

    import { StatsCollector } from "./class/StatsCollector.class.ts";

    import { RegistryTokenStore } from "./registry/RegistryTokenStore.ts";

    import { TempDirectory } from "./class/TempDirectory.class.ts";

    import { Logger, ScannerLoggerEvents } from "./class/logger.class.ts";

    @@ -94,7 +97,6 @@ type InitialPayload =
  
      Partial<Payload> &

      {

        rootDependency: Payload["rootDependency"];

        metadata: Payload["metadata"];

      };

    export async function depWalker(

    @@ -113,7 +115,7 @@ export async function depWalker(
  
        npmRcConfig

      } = options;

      const startedAt = Date.now();

      const statsCollector = new StatsCollector();

      const isRemoteScanning = typeof location === "undefined";

      const tokenStore = new RegistryTokenStore(npmRcConfig, NPM_TOKEN.token);

    @@ -130,18 +132,36 @@ export async function depWalker(
  
        },

        scannerVersion: packageVersion,

        vulnerabilityStrategy,

        warnings: [],

        metadata: {

          startedAt,

          executionTime: 0

        }

        warnings: []

      };

      const dependencies: Map<string, Dependency> = new Map();

      const highlightedPackages: Set<string> = new Set();

      const npmTreeWalker = new npm.TreeWalker({

        registry

        registry,

        providers: {

          pacote: {

            manifest: (spec, opts) => statsCollector.track(`pacote.manifest ${spec}`, () => pacote.manifest(spec, opts)),

            packument: (spec, opts) => statsCollector.track(`pacote.packument ${spec}`, () => pacote.packument(spec, opts))

          }

        }

      });

      const npmApiClient: NpmApiClient = {

        packument: (name, opts) => statsCollector.track(

          `npmRegistrySDK.packument ${name}`,

          () => npmRegistrySDK.packument(name, opts)

        ),

        packumentVersion: (name, version, opts) => statsCollector.track(

          `npmRegistrySDK.packumentVersion ${name}@${version}`,

          () => npmRegistrySDK.packumentVersion(name, version, opts)

        ),

        org: (namespace) => statsCollector.track(

          `npmRegistrySDK.org ${namespace}`,

          () => npmRegistrySDK.org(namespace)

        )

      };

      {

        logger

          .start(ScannerLoggerEvents.analysis.tree)

    @@ -181,7 +201,8 @@ export async function depWalker(
  
            operationsQueue.push(

              new NpmRegistryProvider(name, version, {

                registry,

                tokenStore

                tokenStore,

                npmApiClient

              }).enrichDependencyVersion(dep, dependencyConfusionWarnings, org)

            );

    @@ -350,7 +371,7 @@ export async function depWalker(
  
          packages: [...highlightedPackages]

        };

        payload.dependencies = Object.fromEntries(dependencies);

        payload.metadata.executionTime = Date.now() - startedAt;

        payload.metadata = statsCollector.getStats();

        return payload as Payload;

      }

workspaces/scanner/src/registry/NpmRegistryProvider.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -11,7 +11,8 @@ import * as i18n from "@nodesecure/i18n"; @@
     import { isHTTPError } from "@openally/httpie";
     // Import Internal Dependencies
-    import { PackumentExtractor, type DateProvider } from "./PackumentExtractor.ts";
+    import { PackumentExtractor } from "./PackumentExtractor.ts";
+    import type { DateProvider } from "../class/DateProvider.class.ts";
     import { fetchNpmAvatars } from "./fetchNpmAvatars.ts";
     import type {
       Dependency,
@@ Expand Down @@

workspaces/scanner/src/registry/PackumentExtractor.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -6,10 +6,7 @@ import { packageJSONIntegrityHash } from "@nodesecure/mama"; @@
     import type {
       Dependency
     } from "../types.ts";
-    export interface DateProvider {
-      oneYearAgo(): Date;
-    }
+    import { SystemDateProvider, type DateProvider } from "../class/DateProvider.class.ts";
     export interface PackumentExtractorOptions {
       dateProvider?: DateProvider;
@@ Expand Down Expand Up / @@ -103,12 +100,3 @@ export class PackumentExtractor { @@
         return result;
       }
     }
-    class SystemDateProvider implements DateProvider {
-      oneYearAgo(): Date {
-        const date = new Date();
-        date.setFullYear(date.getFullYear() - 1);
-        return date;
-      }
-    }

workspaces/scanner/src/types.ts

-Original file line number
+Diff line change
@@ Expand Up / @@ -184,6 +184,40 @@ export type GlobalWarning = { message: string; } & ( @@
       |
       DependencyConfusionWarning);
+    export type ApiStats = {
+      /**
+        * UNIX Timestamp just before the api call start
+      */
+      startedAt: number;
+      /**
+        * Execution time in milliseconds
+      */
+      executionTime: number;
+      /**
+        * Name of the api call
+      */
+      name: string;
+    };
+    export type Stats = {
+      /**
+        * UNIX Timestamp when the scan started
+      */
+      startedAt: number;
+      /**
+        * Execution time in milliseconds
+      */
+      executionTime: number;
+      /**
+        * Number of external API calls
+      */
+      apiCallsCount: number;
+      apiCalls: ApiStats[];
+    };
     export interface Payload {
       /** Payload unique id */
       id: string;
@@ Expand All / @@ -207,16 +241,7 @@ export interface Payload { @@
       /** Vulnerability strategy name (npm, snyk, node) */
       vulnerabilityStrategy: Vulnera.Kind;
-      metadata: {
-        /**
-         * UNIX Timestamp when the scan started
-         */
-        startedAt: number;
-        /**
-         * Execution time in milliseconds
-         */
-        executionTime: number;
-      };
+      metadata: Stats;
     }
     export type SemverRange = string | "*";
@@ Expand Down @@

workspaces/scanner/test/StatsCollector.spec.ts

-Original file line number
+Diff line change
@@ -0,0 +1,100 @@
+    // Import Node.js Dependencies
+    import { describe, it } from "node:test";
+    import assert from "node:assert";
+    // Import Internal Dependencies
+    import type { DateProvider } from "../src/class/DateProvider.class.ts";
+    import { StatsCollector } from "../src/class/StatsCollector.class.ts";
+    describe("StatsCollectors", () => {
+      it("should get the expected global start and execution time", () => {
+        const dateProvider = new FakeDateProvider();
+        dateProvider.setNow(1658512000000);
+        const statsCollector = new StatsCollector(dateProvider);
+        dateProvider.setNow(1658512001000);
+        const { startedAt, executionTime } = statsCollector.getStats();
+        assert.strictEqual(startedAt, 1658512000000);
+        assert.strictEqual(executionTime, 1000);
+      });
+      it("should still record the exexution time if the function being tracked throws", () => {
+        const dateProvider = new FakeDateProvider();
+        dateProvider.setNow(1658512000000);
+        const statsCollector = new StatsCollector(dateProvider);
+        assert.throws(() => {
+          statsCollector.track("api/test/1", () => {
+            dateProvider.setNow(1658512001000);
+            throw new Error("oh no!");
+          });
+        });
+        const { apiCalls, apiCallsCount } = statsCollector.getStats();
+        assert.strictEqual(apiCallsCount, 1);
+        assert.deepEqual(apiCalls, [
+          {
+            name: "api/test/1",
+            startedAt: 1658512000000,
+            executionTime: 1000
+          }
+        ]);
+      });
+      it("should be able to track the start and execution time of external api call", async() => {
+        let hasFnOneBeenCalled = false;
+        let hasFnTwoBeenCalled = false;
+        const dateProvider = new FakeDateProvider();
+        dateProvider.setNow(1658512000000);
+        const statsCollector = new StatsCollector(dateProvider);
+        dateProvider.setNow(1658512001001);
+        const promise = statsCollector.track("api/test/1", () => {
+          hasFnOneBeenCalled = true;
+          return Promise.resolve(1);
+        });
+        dateProvider.setNow(1658512002000);
+        const promiseResult = await promise;
+        dateProvider.setNow(1658512003000);
+        const fnResult = statsCollector.track("api/test/2", () => {
+          hasFnTwoBeenCalled = true;
+          dateProvider.setNow(1658512004000);
+          return null;
+        });
+        dateProvider.setNow(1658512005000);
+        const { apiCalls, apiCallsCount } = statsCollector.getStats();
+        assert.strictEqual(promiseResult, 1);
+        assert.strictEqual(fnResult, null);
+        assert.strictEqual(hasFnOneBeenCalled, true);
+        assert.strictEqual(hasFnTwoBeenCalled, true);
+        assert.strictEqual(apiCallsCount, 2);
+        assert.deepEqual(apiCalls, [
+          {
+            name: "api/test/1",
+            startedAt: 1658512001001,
+            executionTime: 999
+          },
+          {
+            name: "api/test/2",
+            startedAt: 1658512003000,
+            executionTime: 1000
+          }
+        ]);
+      });
+    });
+    class FakeDateProvider implements DateProvider {
+      #now: number;
+      now(): number {
+        return this.#now;
+      }
+      oneYearAgo(): Date {
+        return new Date(Date.now() - (365 * 24 * 60 * 60 * 1000));
+      }
+      setNow(now: number) {
+        this.#now = now;
+      }
+    }

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat(scanner): integrate buit-in stats in the response of depWalker #612

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Uh oh!